md-semantic-search 1.0.3 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +132 -16
  2. package/RESEARCH.md +207 -3
  3. package/bin/cli.mjs +449 -13
  4. package/dist/binary-format.js +9 -0
  5. package/dist/completions.js +11 -1
  6. package/dist/core.d.ts +42 -4
  7. package/dist/core.js +208 -14
  8. package/dist/federation.d.ts +16 -1
  9. package/dist/federation.js +33 -16
  10. package/dist/filter.d.ts +7 -0
  11. package/dist/filter.js +13 -0
  12. package/dist/hnsw.d.ts +96 -0
  13. package/dist/hnsw.js +326 -0
  14. package/dist/index-format.js +8 -1
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +4 -4
  17. package/dist/indexer.d.ts +18 -1
  18. package/dist/indexer.js +92 -18
  19. package/dist/ivf.d.ts +14 -7
  20. package/dist/ivf.js +126 -18
  21. package/dist/lexical.d.ts +10 -0
  22. package/dist/lexical.js +98 -15
  23. package/dist/markdown-parser.d.ts +62 -0
  24. package/dist/markdown-parser.js +227 -30
  25. package/dist/mcp.d.ts +229 -7
  26. package/dist/mcp.js +743 -44
  27. package/dist/models.d.ts +8 -2
  28. package/dist/models.js +84 -32
  29. package/dist/providers.d.ts +56 -0
  30. package/dist/providers.js +305 -0
  31. package/dist/quantization.d.ts +12 -0
  32. package/dist/quantization.js +54 -0
  33. package/dist/rag.d.ts +132 -7
  34. package/dist/rag.js +314 -38
  35. package/dist/repair.d.ts +1 -1
  36. package/dist/repair.js +53 -2
  37. package/dist/rerank.d.ts +51 -6
  38. package/dist/rerank.js +69 -10
  39. package/dist/search.d.ts +49 -3
  40. package/dist/search.js +387 -78
  41. package/dist/serve.d.ts +35 -0
  42. package/dist/serve.js +574 -58
  43. package/dist/tui.d.ts +14 -0
  44. package/dist/tui.js +132 -15
  45. package/dist/watcher.js +23 -12
  46. package/dist/webui.d.ts +3 -3
  47. package/dist/webui.js +225 -5
  48. package/dist/wikilinks.d.ts +31 -0
  49. package/dist/wikilinks.js +185 -1
  50. package/package.json +3 -3
  51. package/src/binary-format.ts +13 -0
  52. package/src/completions.ts +11 -1
  53. package/src/core.ts +249 -13
  54. package/src/federation.ts +51 -17
  55. package/src/filter.ts +14 -0
  56. package/src/hnsw.ts +414 -0
  57. package/src/index-format.ts +7 -1
  58. package/src/index.ts +7 -2
  59. package/src/indexer.ts +112 -19
  60. package/src/ivf.ts +157 -20
  61. package/src/lexical.ts +91 -13
  62. package/src/markdown-parser.ts +279 -37
  63. package/src/mcp.ts +820 -47
  64. package/src/models.ts +97 -35
  65. package/src/providers.ts +345 -0
  66. package/src/quantization.ts +61 -0
  67. package/src/rag.ts +451 -34
  68. package/src/repair.ts +51 -2
  69. package/src/rerank.ts +121 -14
  70. package/src/search.ts +424 -82
  71. package/src/serve.ts +589 -66
  72. package/src/tui.ts +142 -16
  73. package/src/watcher.ts +21 -10
  74. package/src/webui.ts +225 -5
  75. package/src/wikilinks.ts +220 -1
package/src/rag.ts CHANGED
@@ -1,12 +1,9 @@
1
- /**
2
- * Offline QA & Local RAG Answer Synthesis (issue #101).
3
- *
4
- * Implements grounded question answering with strict provenance citations ([file#heading]),
5
- * supporting local LLM bridges (Ollama / LocalAI / OpenAI-compatible endpoint) and a
6
- * fast built-in zero-dependency extractive synthesis fallback.
7
- */
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import crypto from 'node:crypto';
8
4
  import { searchIndex, loadIndex, LoadedIndex, SearchResultHit } from './search.js';
9
5
  import { tokenize } from './lexical.js';
6
+ import { ChatProviderId, ResolvedChat, resolveChatProvider } from './providers.js';
10
7
 
11
8
  export interface Citation {
12
9
  file: string;
@@ -17,7 +14,56 @@ export interface Citation {
17
14
  export interface SynthesisResult {
18
15
  answer: string;
19
16
  citations: Citation[];
17
+ /**
18
+ * issue #199 follow-up: citations the answer actually mentions (bracket
19
+ * formats `[file#heading]` / `[file › heading]` / `[file]`), in order of
20
+ * first mention. Deterministic post-processing of the LLM text — a subset
21
+ * of `citations`, which stays the full retrieval manifest for compatibility.
22
+ */
23
+ citationsUsed?: Citation[];
20
24
  passages: SearchResultHit[];
25
+ /** Which synthesis stage produced the answer (issue #197 observability). */
26
+ mode?: 'llm' | 'endpoint' | 'extractive';
27
+ }
28
+
29
+ /**
30
+ * System/user split for transports that keep the two apart on the wire —
31
+ * MCP `sampling/createMessage` (issue #197) carries a `systemPrompt` field
32
+ * and a message list, unlike the single-string local LLM bridge.
33
+ */
34
+ export interface LlmStructuredRequest {
35
+ system: string;
36
+ user: string;
37
+ }
38
+
39
+ export interface ChatSourceManifestItem {
40
+ file: string;
41
+ heading?: string;
42
+ headingPath?: string[];
43
+ startLine?: number;
44
+ endLine?: number;
45
+ chunkHash?: string;
46
+ score: number;
47
+ }
48
+
49
+ export interface ChatTurn {
50
+ id: string;
51
+ timestamp: string;
52
+ query: string;
53
+ answer: string;
54
+ citations: Citation[];
55
+ manifest: ChatSourceManifestItem[];
56
+ /** Which synthesis stage produced the answer (issue #199; optional in persisted sessions). */
57
+ mode?: SynthesisResult['mode'];
58
+ }
59
+
60
+ export interface ChatSession {
61
+ id: string;
62
+ createdAt: string;
63
+ updatedAt: string;
64
+ db?: string;
65
+ model?: string;
66
+ turns: ChatTurn[];
21
67
  }
22
68
 
23
69
  export interface AskOptions {
@@ -27,11 +73,43 @@ export interface AskOptions {
27
73
  cacheDir?: string;
28
74
  db?: string;
29
75
  k?: number;
76
+ offline?: boolean;
77
+ llmEndpoint?: string;
78
+ llmModel?: string;
79
+ /** issue #199: chat wire format for llmEndpoint — 'openai' (default) or 'ollama'. */
80
+ llmProvider?: ChatProviderId;
81
+ llmFn?: (prompt: string) => Promise<string>;
82
+ llmRequestFn?: (req: LlmStructuredRequest) => Promise<string>;
83
+ /** issue #199: incremental render hook — fires per token chunk while the LLM streams. */
84
+ onDelta?: (delta: string) => void;
85
+ /** issue #199: injection seam for tests/serve; built from llmEndpoint when absent. */
86
+ llmChat?: ResolvedChat;
87
+ systemPrompt?: string;
88
+ embedFn?: any;
89
+ }
90
+
91
+ export interface ChatTurnOptions {
92
+ session?: ChatSession;
93
+ query: string;
94
+ loaded?: LoadedIndex;
95
+ indexDir?: string;
96
+ cacheDir?: string;
97
+ db?: string;
98
+ k?: number;
99
+ offline?: boolean;
30
100
  llmEndpoint?: string;
31
101
  llmModel?: string;
102
+ /** issue #199: chat wire format for llmEndpoint — 'openai' (default) or 'ollama'. */
103
+ llmProvider?: ChatProviderId;
32
104
  llmFn?: (prompt: string) => Promise<string>;
105
+ llmRequestFn?: (req: LlmStructuredRequest) => Promise<string>;
106
+ /** issue #199: incremental render hook — fires per token chunk while the LLM streams. */
107
+ onDelta?: (delta: string) => void;
108
+ /** issue #199: injection seam for tests/serve; built from llmEndpoint when absent. */
109
+ llmChat?: ResolvedChat;
33
110
  systemPrompt?: string;
34
111
  embedFn?: any;
112
+ log?: (msg: string) => void;
35
113
  }
36
114
 
37
115
  export const DEFAULT_RAG_SYSTEM_PROMPT =
@@ -39,6 +117,108 @@ export const DEFAULT_RAG_SYSTEM_PROMPT =
39
117
  'Always cite your sources using [file#heading] format. ' +
40
118
  'If the context does not contain the answer, say that the notes do not have enough information.';
41
119
 
120
+ /**
121
+ * Returns the directory where chat sessions are persisted.
122
+ */
123
+ export function getSessionsDir(indexDir: string): string {
124
+ return path.join(indexDir, 'sessions');
125
+ }
126
+
127
+ const SESSION_ID_PATTERN = /^[a-zA-Z0-9_-]{1,64}$/;
128
+
129
+ /**
130
+ * Validates a session id before it is used as a filesystem path component.
131
+ */
132
+ export function assertSafeSessionId(id: string): string {
133
+ if (!SESSION_ID_PATTERN.test(id)) {
134
+ throw new Error('Invalid session ID: expected 1-64 letters, numbers, hyphens, or underscores');
135
+ }
136
+ return id;
137
+ }
138
+
139
+ /**
140
+ * Atomically saves a ChatSession to disk (<indexDir>/sessions/<id>.json).
141
+ */
142
+ export function saveChatSession(indexDir: string, session: ChatSession): string {
143
+ assertSafeSessionId(session.id);
144
+ const dir = getSessionsDir(indexDir);
145
+ fs.mkdirSync(dir, { recursive: true });
146
+ const targetPath = path.join(dir, `${session.id}.json`);
147
+ const tmpPath = path.join(dir, `${session.id}.${Date.now()}.tmp`);
148
+ fs.writeFileSync(tmpPath, JSON.stringify(session, null, 2), 'utf8');
149
+ fs.renameSync(tmpPath, targetPath);
150
+ return targetPath;
151
+ }
152
+
153
+ /**
154
+ * Loads a ChatSession by id or creates a new one. Degrades safely if corrupt.
155
+ */
156
+ export function loadChatSession(
157
+ indexDir: string,
158
+ id: string,
159
+ log: (msg: string) => void = () => {}
160
+ ): ChatSession {
161
+ assertSafeSessionId(id);
162
+ const sessionPath = path.join(getSessionsDir(indexDir), `${id}.json`);
163
+ if (fs.existsSync(sessionPath)) {
164
+ try {
165
+ const raw = fs.readFileSync(sessionPath, 'utf8');
166
+ const parsed = JSON.parse(raw);
167
+ if (parsed && typeof parsed.id === 'string' && Array.isArray(parsed.turns)) {
168
+ return parsed as ChatSession;
169
+ }
170
+ log(`warning: session ${id} is malformed; starting a fresh session.`);
171
+ } catch (e: any) {
172
+ log(`warning: failed to parse session ${id} (${e.message}); starting fresh.`);
173
+ }
174
+ }
175
+
176
+ const now = new Date().toISOString();
177
+ return {
178
+ id,
179
+ createdAt: now,
180
+ updatedAt: now,
181
+ turns: [],
182
+ };
183
+ }
184
+
185
+ /**
186
+ * Lists all existing ChatSessions in the indexDir, sorted by updatedAt descending.
187
+ */
188
+ export function listChatSessions(
189
+ indexDir: string
190
+ ): Array<{ id: string; createdAt: string; updatedAt: string; turnsCount: number }> {
191
+ const dir = getSessionsDir(indexDir);
192
+ if (!fs.existsSync(dir)) return [];
193
+
194
+ const out: Array<{ id: string; createdAt: string; updatedAt: string; turnsCount: number }> = [];
195
+ let entries: string[];
196
+ try {
197
+ entries = fs.readdirSync(dir);
198
+ } catch {
199
+ return [];
200
+ }
201
+
202
+ for (const file of entries) {
203
+ if (!file.endsWith('.json')) continue;
204
+ const p = path.join(dir, file);
205
+ try {
206
+ const raw = fs.readFileSync(p, 'utf8');
207
+ const s = JSON.parse(raw);
208
+ if (s && typeof s.id === 'string' && Array.isArray(s.turns)) {
209
+ out.push({
210
+ id: s.id,
211
+ createdAt: s.createdAt || s.updatedAt || '',
212
+ updatedAt: s.updatedAt || s.createdAt || '',
213
+ turnsCount: s.turns.length,
214
+ });
215
+ }
216
+ } catch {}
217
+ }
218
+
219
+ return out.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
220
+ }
221
+
42
222
  /**
43
223
  * Built-in extractive answer synthesizer for fully offline, zero-dependency QA.
44
224
  */
@@ -88,6 +268,55 @@ export function extractAnswerFallback(query: string, passages: SearchResultHit[]
88
268
  return `Based on your notes:\n\n${body}`;
89
269
  }
90
270
 
271
+ const CITATION_MENTION_RE = /\[([^\]\n]{1,200})\]/g;
272
+
273
+ /** `[file]` | `[file#heading]` | `[file › heading]` → {file, heading?}; null for non-citations. */
274
+ function parseCitationMention(text: string): { file: string; heading?: string } | null {
275
+ const t = text.trim();
276
+ if (!t) return null;
277
+ let file: string;
278
+ let heading: string | undefined;
279
+ const hash = t.indexOf('#');
280
+ const sep = t.indexOf('›');
281
+ if (hash > 0) {
282
+ file = t.slice(0, hash).trim();
283
+ heading = t.slice(hash + 1).trim();
284
+ } else if (sep > 0) {
285
+ file = t.slice(0, sep).trim();
286
+ heading = t.slice(sep + 1).trim();
287
+ } else {
288
+ file = t;
289
+ }
290
+ // markdown links, footnote refs, and empty brackets are not citations
291
+ if (!file || /https?:\/\//i.test(file)) return null;
292
+ return { file, heading: heading || undefined };
293
+ }
294
+
295
+ /**
296
+ * issue #199 follow-up: which of the retrieved citations the answer text
297
+ * actually cites. Bracket-only scan (the exact formats the system prompt and
298
+ * the extractive fallback emit), exact file/heading match against the
299
+ * retrieval manifest, order of first mention. No mention → [].
300
+ */
301
+ export function extractUsedCitations(answer: string, citations: Citation[]): Citation[] {
302
+ const used: Citation[] = [];
303
+ const seen = new Set<string>();
304
+ for (const m of answer.matchAll(CITATION_MENTION_RE)) {
305
+ const mention = parseCitationMention(m[1]);
306
+ if (!mention) continue;
307
+ const hit = citations.find(
308
+ (c) => c.file === mention.file && (mention.heading === undefined || c.heading === mention.heading)
309
+ );
310
+ if (!hit) continue;
311
+ const key = `${hit.file}\u0000${hit.heading ?? ''}`;
312
+ if (!seen.has(key)) {
313
+ seen.add(key);
314
+ used.push(hit);
315
+ }
316
+ }
317
+ return used;
318
+ }
319
+
91
320
  /**
92
321
  * Synthesizes an answer from retrieved passages using an LLM or fallback.
93
322
  */
@@ -96,9 +325,18 @@ export async function synthesizeAnswer(
96
325
  passages: SearchResultHit[],
97
326
  options: {
98
327
  llmFn?: (prompt: string) => Promise<string>;
328
+ llmRequestFn?: (req: LlmStructuredRequest) => Promise<string>;
99
329
  llmEndpoint?: string;
100
330
  llmModel?: string;
331
+ /** issue #199: chat wire format for llmEndpoint — 'openai' (default) or 'ollama'. */
332
+ llmProvider?: ChatProviderId;
333
+ llmChat?: ResolvedChat;
334
+ /** issue #199: incremental render hook — fires per token chunk while the LLM streams. */
335
+ onDelta?: (delta: string) => void;
336
+ /** Cooperative cancel (serve /ask: client disconnected mid-stream). */
337
+ signal?: AbortSignal;
101
338
  systemPrompt?: string;
339
+ conversationHistory?: string;
102
340
  } = {}
103
341
  ): Promise<SynthesisResult> {
104
342
  const citations: Citation[] = passages.map((p) => ({
@@ -107,46 +345,73 @@ export async function synthesizeAnswer(
107
345
  score: p.score,
108
346
  }));
109
347
 
110
- if (options.llmFn) {
111
- const contextBlocks = passages.map((p, i) => {
348
+ // One prompt assembly for every stage: `system + user` reproduces the
349
+ // historical single-string prompt byte-for-byte; the split serves the
350
+ // structured transports (MCP sampling, issue #197).
351
+ const system = options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT;
352
+ const historyHeader = options.conversationHistory ? `\nConversation History:\n${options.conversationHistory}\n` : '';
353
+ const contextBlocks = passages
354
+ .map((p, i) => {
112
355
  const header = p.heading ? `${p.file} › ${p.heading}` : p.file;
113
356
  return `[Source ${i + 1}: ${header}]\n${p.snippet}`;
114
- }).join('\n\n---\n\n');
357
+ })
358
+ .join('\n\n---\n\n');
359
+ const user = `${historyHeader}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
360
+
361
+ // every stage returns through one shape so citationsUsed stays consistent
362
+ const finish = (answer: string, mode: SynthesisResult['mode']): SynthesisResult => ({
363
+ answer,
364
+ citations,
365
+ citationsUsed: extractUsedCitations(answer, citations),
366
+ passages,
367
+ mode,
368
+ });
115
369
 
116
- const prompt = `${options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
117
- const answer = await options.llmFn(prompt);
118
- return { answer, citations, passages };
370
+ // issue #197: structured hook (MCP sampling). Best-effort like the endpoint
371
+ // path — a refusal or timeout falls through to the extractive answer.
372
+ if (options.llmRequestFn) {
373
+ try {
374
+ const answer = await options.llmRequestFn({ system, user });
375
+ if (answer) return finish(answer, 'llm');
376
+ } catch {
377
+ // Fall through to extractive.
378
+ }
119
379
  }
120
380
 
121
- if (options.llmEndpoint) {
122
- const contextBlocks = passages.map((p, i) => {
123
- const header = p.heading ? `${p.file} › ${p.heading}` : p.file;
124
- return `[Source ${i + 1}: ${header}]\n${p.snippet}`;
125
- }).join('\n\n---\n\n');
381
+ if (options.llmFn) {
382
+ const answer = await options.llmFn(system + user);
383
+ return finish(answer, 'llm');
384
+ }
126
385
 
127
- const prompt = `${options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
386
+ // issue #199: unified chat provider path. `/chat/completions` by default
387
+ // (same wire format HyDE uses — llama.cpp, LM Studio, vLLM, Ollama /v1 all
388
+ // speak it), Ollama-native `/api/generate` under `--llm-provider ollama`.
389
+ // Streaming deltas flow to onDelta; the full text is assembled for the
390
+ // returned result. Best-effort like the old endpoint path — a refusal or
391
+ // idle timeout falls through to the extractive answer.
392
+ if (options.llmChat || options.llmEndpoint) {
393
+ const chat =
394
+ options.llmChat ??
395
+ resolveChatProvider({
396
+ provider: options.llmProvider ?? 'openai',
397
+ model: options.llmModel,
398
+ baseUrl: options.llmEndpoint,
399
+ });
128
400
  try {
129
- const resp = await fetch(`${options.llmEndpoint}/api/generate`, {
130
- method: 'POST',
131
- headers: { 'Content-Type': 'application/json' },
132
- body: JSON.stringify({
133
- model: options.llmModel || 'llama3',
134
- prompt,
135
- stream: false,
136
- }),
401
+ const answer = await chat.complete({
402
+ system,
403
+ user,
404
+ ...(options.onDelta ? { onDelta: options.onDelta } : {}),
405
+ ...(options.signal ? { signal: options.signal } : {}),
137
406
  });
138
- if (resp.ok) {
139
- const data: any = await resp.json();
140
- const answer = data.response || data.text || '';
141
- if (answer) return { answer, citations, passages };
142
- }
407
+ if (answer) return finish(answer, 'endpoint');
143
408
  } catch {
144
- // Fallback to extractive
409
+ // Fall through to extractive.
145
410
  }
146
411
  }
147
412
 
148
413
  const answer = extractAnswerFallback(query, passages);
149
- return { answer, citations, passages };
414
+ return finish(answer, 'extractive');
150
415
  }
151
416
 
152
417
  /**
@@ -160,8 +425,12 @@ export async function askQuestion(opts: AskOptions): Promise<SynthesisResult> {
160
425
  cacheDir = '',
161
426
  k = 5,
162
427
  llmFn,
428
+ llmRequestFn,
163
429
  llmEndpoint,
164
430
  llmModel,
431
+ llmProvider,
432
+ llmChat,
433
+ onDelta,
165
434
  systemPrompt,
166
435
  embedFn,
167
436
  } = opts;
@@ -178,12 +447,160 @@ export async function askQuestion(opts: AskOptions): Promise<SynthesisResult> {
178
447
  query,
179
448
  k,
180
449
  embedFn,
450
+ offline: opts.offline,
181
451
  });
182
452
 
183
453
  return synthesizeAnswer(query, passages, {
184
454
  llmFn,
455
+ llmRequestFn,
185
456
  llmEndpoint,
186
457
  llmModel,
458
+ llmProvider,
459
+ llmChat,
460
+ onDelta,
187
461
  systemPrompt,
188
462
  });
189
463
  }
464
+
465
+ /**
466
+ * Executes a single conversational turn in a ChatSession with context carry-over and source manifest.
467
+ */
468
+ export async function chatTurn(opts: ChatTurnOptions): Promise<{
469
+ session: ChatSession;
470
+ turn: ChatTurn;
471
+ answer: string;
472
+ citations: Citation[];
473
+ manifest: ChatSourceManifestItem[];
474
+ /** Which synthesis stage produced the answer ('llm'/'endpoint'/'extractive'). */
475
+ mode: SynthesisResult['mode'];
476
+ /** Citations the answer text actually mentions (issue #199 follow-up). */
477
+ citationsUsed: Citation[];
478
+ }> {
479
+ const {
480
+ query,
481
+ loaded: inputLoaded,
482
+ indexDir,
483
+ cacheDir = '',
484
+ k = 5,
485
+ llmFn,
486
+ llmRequestFn,
487
+ llmEndpoint,
488
+ llmModel,
489
+ llmProvider,
490
+ llmChat,
491
+ onDelta,
492
+ systemPrompt,
493
+ embedFn,
494
+ log = () => {},
495
+ } = opts;
496
+
497
+ let loaded = inputLoaded;
498
+ if (!loaded) {
499
+ if (!indexDir) throw new Error('chatTurn requires either loaded index or indexDir');
500
+ loaded = loadIndex(indexDir);
501
+ }
502
+
503
+ let session = opts.session;
504
+ if (!session) {
505
+ const newId = crypto.randomUUID().slice(0, 8);
506
+ session = indexDir ? loadChatSession(indexDir, newId, log) : {
507
+ id: newId,
508
+ createdAt: new Date().toISOString(),
509
+ updatedAt: new Date().toISOString(),
510
+ turns: [],
511
+ };
512
+ }
513
+
514
+ // 1. Build contextual retrieval query for follow-up questions
515
+ let retrievalQuery = query;
516
+ if (session.turns.length > 0) {
517
+ const prevTurn = session.turns[session.turns.length - 1];
518
+ const prevTokens = tokenize(prevTurn.query).filter((t) => t.length > 3).slice(0, 4);
519
+ if (prevTokens.length > 0) {
520
+ retrievalQuery = `${query} ${prevTokens.join(' ')}`.trim();
521
+ }
522
+ }
523
+
524
+ // 2. Search index
525
+ const passages = await searchIndex({
526
+ loaded,
527
+ cacheDir,
528
+ query: retrievalQuery,
529
+ k,
530
+ embedFn,
531
+ offline: opts.offline,
532
+ });
533
+
534
+ // 3. Construct source manifest from retrieved chunks
535
+ const manifest: ChatSourceManifestItem[] = passages.map((p) => {
536
+ const chunk = loaded.index?.chunks?.find(
537
+ (c: any) => c.file === p.file && (c.heading === p.heading || (!c.heading && !p.heading))
538
+ );
539
+ return {
540
+ file: p.file,
541
+ heading: p.heading,
542
+ headingPath: chunk?.headingPath,
543
+ startLine: chunk?.startLine,
544
+ endLine: chunk?.endLine,
545
+ chunkHash: chunk?.chunkHash,
546
+ score: p.score,
547
+ };
548
+ });
549
+
550
+ // 4. Conversation history string for LLM prompt
551
+ let conversationHistory = '';
552
+ if (session.turns.length > 0) {
553
+ conversationHistory = session.turns
554
+ .slice(-3)
555
+ .map((t) => `User: ${t.query}\nAssistant: ${t.answer}`)
556
+ .join('\n\n');
557
+ }
558
+
559
+ // 5. Synthesize answer
560
+ const synth = await synthesizeAnswer(query, passages, {
561
+ llmFn,
562
+ llmRequestFn,
563
+ llmEndpoint,
564
+ llmModel,
565
+ llmProvider,
566
+ llmChat,
567
+ onDelta,
568
+ systemPrompt,
569
+ conversationHistory,
570
+ });
571
+
572
+ // 6. Append turn to session
573
+ const turn: ChatTurn = {
574
+ id: crypto.randomUUID(),
575
+ timestamp: new Date().toISOString(),
576
+ query,
577
+ answer: synth.answer,
578
+ citations: synth.citations,
579
+ manifest,
580
+ mode: synth.mode,
581
+ };
582
+
583
+ session.turns.push(turn);
584
+ session.updatedAt = turn.timestamp;
585
+ if (loaded.index?.db) session.db = loaded.index.db;
586
+ if (loaded.model?.id) session.model = loaded.model.id;
587
+
588
+ if (indexDir) {
589
+ try {
590
+ saveChatSession(indexDir, session);
591
+ } catch (e: any) {
592
+ log(`warning: failed to save chat session ${session.id}: ${e.message}`);
593
+ }
594
+ }
595
+
596
+ return {
597
+ session,
598
+ turn,
599
+ answer: turn.answer,
600
+ citations: turn.citations,
601
+ manifest: turn.manifest,
602
+ mode: turn.mode ?? 'extractive',
603
+ citationsUsed: extractUsedCitations(turn.answer, turn.citations),
604
+ };
605
+ }
606
+
package/src/repair.ts CHANGED
@@ -20,6 +20,7 @@ export interface RepairAction {
20
20
  | 'remove-stale-lock'
21
21
  | 'remove-broken-vectors-bin'
22
22
  | 'remove-corrupt-ivf'
23
+ | 'remove-corrupt-hnsw'
23
24
  | 'rebuild-index'
24
25
  | 'rewrite-hashes';
25
26
  detail: string;
@@ -130,7 +131,7 @@ export function planRepairs(indexDir: string, opts: ProbeOptions = {}): RepairPl
130
131
  }
131
132
  }
132
133
 
133
- // --- 3. corrupt ivf.json (optional ANN sidecar; loader silently ignores it) ---
134
+ // --- 3. corrupt ANN sidecars (optional; loader silently ignores them) ---
134
135
  const ivfPath = path.join(indexDir, 'ivf.json');
135
136
  const ivfRaw = readJsonIfParsable(ivfPath);
136
137
  if (ivfRaw.ok === false) {
@@ -139,12 +140,22 @@ export function planRepairs(indexDir: string, opts: ProbeOptions = {}): RepairPl
139
140
  detail: 'ivf.json is not valid JSON — removing it re-enables brute-force ANN fallback',
140
141
  });
141
142
  }
143
+ // issue #201: hnsw.json gets the same corrupt-sidecar treatment
144
+ const hnswPath = path.join(indexDir, 'hnsw.json');
145
+ const hnswRaw = readJsonIfParsable(hnswPath);
146
+ if (hnswRaw.ok === false) {
147
+ plan.actions.push({
148
+ action: 'remove-corrupt-hnsw',
149
+ detail: 'hnsw.json is not valid JSON — removing it re-enables brute-force ANN fallback',
150
+ });
151
+ }
142
152
 
143
153
  // --- 4. corrupt vectors.json ---
144
154
  const liveWriter = plan.skipped.some(
145
155
  (s) => s.action === 'remove-stale-lock' && /live writer/.test(s.detail),
146
156
  );
147
- const vectorsRaw = readJsonIfParsable(path.join(indexDir, 'vectors.json'));
157
+ const vectorsPath = path.join(indexDir, 'vectors.json');
158
+ const vectorsRaw = readJsonIfParsable(vectorsPath);
148
159
  if (vectorsRaw.ok === false) {
149
160
  if (db && !liveWriter) {
150
161
  plan.actions.push({ action: 'rebuild-index', detail: `vectors.json is not valid JSON — rebuild from ${db}` });
@@ -161,6 +172,41 @@ export function planRepairs(indexDir: string, opts: ProbeOptions = {}): RepairPl
161
172
  }
162
173
  }
163
174
 
175
+ // --- 4b. vectors.json SHA-256 sidecar mismatch (issue #166) ---
176
+ // `mdss check` reports this as an integrity failure; without a planned
177
+ // repair `check --fix` printed "nothing to repair" and left the index
178
+ // broken. Mirror the check's hashing (raw bytes vs first sidecar token)
179
+ // and plan the same rebuild as the corrupt-JSON path — rewriting the
180
+ // sidecar would bless whatever tampered content is on disk.
181
+ if (vectorsRaw.ok === true) {
182
+ const vectorsShaPath = `${vectorsPath}.sha256`;
183
+ if (fs.existsSync(vectorsShaPath)) {
184
+ let mismatch: string | null = null;
185
+ try {
186
+ const actual = crypto.createHash('sha256').update(fs.readFileSync(vectorsPath)).digest('hex');
187
+ const expected = fs.readFileSync(vectorsShaPath, 'utf8').trim().split(/\s+/)[0];
188
+ if (expected && expected !== actual) mismatch = 'SHA-256 mismatch';
189
+ } catch (e: any) {
190
+ mismatch = e.message;
191
+ }
192
+ if (mismatch) {
193
+ if (db && !liveWriter) {
194
+ plan.actions.push({ action: 'rebuild-index', detail: `vectors.json integrity check failed (${mismatch}) — rebuild from ${db}` });
195
+ } else if (liveWriter) {
196
+ plan.skipped.push({
197
+ action: 'rebuild-index',
198
+ detail: `vectors.json integrity check failed (${mismatch}), but a live writer holds the lock — retry after it finishes`,
199
+ });
200
+ } else {
201
+ plan.skipped.push({
202
+ action: 'rebuild-index',
203
+ detail: `vectors.json integrity check failed (${mismatch}) — pass --db so --fix can rebuild it`,
204
+ });
205
+ }
206
+ }
207
+ }
208
+ }
209
+
164
210
  // --- 5. damaged .hashes.json ---
165
211
  const hashesRaw = readJsonIfParsable(path.join(indexDir, '.hashes.json'));
166
212
  if (hashesRaw.ok === false) {
@@ -213,6 +259,9 @@ export function applyRepairs(
213
259
  case 'remove-corrupt-ivf':
214
260
  fs.rmSync(path.join(indexDir, 'ivf.json'), { force: true });
215
261
  break;
262
+ case 'remove-corrupt-hnsw':
263
+ fs.rmSync(path.join(indexDir, 'hnsw.json'), { force: true });
264
+ break;
216
265
  case 'rewrite-hashes': {
217
266
  const hashes = recomputeHashes(opts.db!);
218
267
  fs.writeFileSync(path.join(indexDir, '.hashes.json'), JSON.stringify(hashes, null, 2));