md-semantic-search 1.0.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -16
- package/RESEARCH.md +207 -3
- package/bin/cli.mjs +449 -13
- package/dist/binary-format.js +9 -0
- package/dist/completions.js +11 -1
- package/dist/core.d.ts +42 -4
- package/dist/core.js +208 -14
- package/dist/federation.d.ts +16 -1
- package/dist/federation.js +33 -16
- package/dist/filter.d.ts +7 -0
- package/dist/filter.js +13 -0
- package/dist/hnsw.d.ts +96 -0
- package/dist/hnsw.js +326 -0
- package/dist/index-format.js +8 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +4 -4
- package/dist/indexer.d.ts +18 -1
- package/dist/indexer.js +92 -18
- package/dist/ivf.d.ts +14 -7
- package/dist/ivf.js +126 -18
- package/dist/lexical.d.ts +10 -0
- package/dist/lexical.js +98 -15
- package/dist/markdown-parser.d.ts +62 -0
- package/dist/markdown-parser.js +227 -30
- package/dist/mcp.d.ts +229 -7
- package/dist/mcp.js +743 -44
- package/dist/models.d.ts +8 -2
- package/dist/models.js +84 -32
- package/dist/providers.d.ts +56 -0
- package/dist/providers.js +305 -0
- package/dist/quantization.d.ts +12 -0
- package/dist/quantization.js +54 -0
- package/dist/rag.d.ts +132 -7
- package/dist/rag.js +314 -38
- package/dist/repair.d.ts +1 -1
- package/dist/repair.js +53 -2
- package/dist/rerank.d.ts +51 -6
- package/dist/rerank.js +69 -10
- package/dist/search.d.ts +49 -3
- package/dist/search.js +387 -78
- package/dist/serve.d.ts +35 -0
- package/dist/serve.js +574 -58
- package/dist/tui.d.ts +14 -0
- package/dist/tui.js +132 -15
- package/dist/watcher.js +23 -12
- package/dist/webui.d.ts +3 -3
- package/dist/webui.js +225 -5
- package/dist/wikilinks.d.ts +31 -0
- package/dist/wikilinks.js +185 -1
- package/package.json +3 -3
- package/src/binary-format.ts +13 -0
- package/src/completions.ts +11 -1
- package/src/core.ts +249 -13
- package/src/federation.ts +51 -17
- package/src/filter.ts +14 -0
- package/src/hnsw.ts +414 -0
- package/src/index-format.ts +7 -1
- package/src/index.ts +7 -2
- package/src/indexer.ts +112 -19
- package/src/ivf.ts +157 -20
- package/src/lexical.ts +91 -13
- package/src/markdown-parser.ts +279 -37
- package/src/mcp.ts +820 -47
- package/src/models.ts +97 -35
- package/src/providers.ts +345 -0
- package/src/quantization.ts +61 -0
- package/src/rag.ts +451 -34
- package/src/repair.ts +51 -2
- package/src/rerank.ts +121 -14
- package/src/search.ts +424 -82
- package/src/serve.ts +589 -66
- package/src/tui.ts +142 -16
- package/src/watcher.ts +21 -10
- package/src/webui.ts +225 -5
- package/src/wikilinks.ts +220 -1
package/src/rag.ts
CHANGED
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
* Implements grounded question answering with strict provenance citations ([file#heading]),
|
|
5
|
-
* supporting local LLM bridges (Ollama / LocalAI / OpenAI-compatible endpoint) and a
|
|
6
|
-
* fast built-in zero-dependency extractive synthesis fallback.
|
|
7
|
-
*/
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import crypto from 'node:crypto';
|
|
8
4
|
import { searchIndex, loadIndex, LoadedIndex, SearchResultHit } from './search.js';
|
|
9
5
|
import { tokenize } from './lexical.js';
|
|
6
|
+
import { ChatProviderId, ResolvedChat, resolveChatProvider } from './providers.js';
|
|
10
7
|
|
|
11
8
|
export interface Citation {
|
|
12
9
|
file: string;
|
|
@@ -17,7 +14,56 @@ export interface Citation {
|
|
|
17
14
|
export interface SynthesisResult {
|
|
18
15
|
answer: string;
|
|
19
16
|
citations: Citation[];
|
|
17
|
+
/**
|
|
18
|
+
* issue #199 follow-up: citations the answer actually mentions (bracket
|
|
19
|
+
* formats `[file#heading]` / `[file › heading]` / `[file]`), in order of
|
|
20
|
+
* first mention. Deterministic post-processing of the LLM text — a subset
|
|
21
|
+
* of `citations`, which stays the full retrieval manifest for compatibility.
|
|
22
|
+
*/
|
|
23
|
+
citationsUsed?: Citation[];
|
|
20
24
|
passages: SearchResultHit[];
|
|
25
|
+
/** Which synthesis stage produced the answer (issue #197 observability). */
|
|
26
|
+
mode?: 'llm' | 'endpoint' | 'extractive';
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* System/user split for transports that keep the two apart on the wire —
|
|
31
|
+
* MCP `sampling/createMessage` (issue #197) carries a `systemPrompt` field
|
|
32
|
+
* and a message list, unlike the single-string local LLM bridge.
|
|
33
|
+
*/
|
|
34
|
+
export interface LlmStructuredRequest {
|
|
35
|
+
system: string;
|
|
36
|
+
user: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface ChatSourceManifestItem {
|
|
40
|
+
file: string;
|
|
41
|
+
heading?: string;
|
|
42
|
+
headingPath?: string[];
|
|
43
|
+
startLine?: number;
|
|
44
|
+
endLine?: number;
|
|
45
|
+
chunkHash?: string;
|
|
46
|
+
score: number;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface ChatTurn {
|
|
50
|
+
id: string;
|
|
51
|
+
timestamp: string;
|
|
52
|
+
query: string;
|
|
53
|
+
answer: string;
|
|
54
|
+
citations: Citation[];
|
|
55
|
+
manifest: ChatSourceManifestItem[];
|
|
56
|
+
/** Which synthesis stage produced the answer (issue #199; optional in persisted sessions). */
|
|
57
|
+
mode?: SynthesisResult['mode'];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface ChatSession {
|
|
61
|
+
id: string;
|
|
62
|
+
createdAt: string;
|
|
63
|
+
updatedAt: string;
|
|
64
|
+
db?: string;
|
|
65
|
+
model?: string;
|
|
66
|
+
turns: ChatTurn[];
|
|
21
67
|
}
|
|
22
68
|
|
|
23
69
|
export interface AskOptions {
|
|
@@ -27,11 +73,43 @@ export interface AskOptions {
|
|
|
27
73
|
cacheDir?: string;
|
|
28
74
|
db?: string;
|
|
29
75
|
k?: number;
|
|
76
|
+
offline?: boolean;
|
|
77
|
+
llmEndpoint?: string;
|
|
78
|
+
llmModel?: string;
|
|
79
|
+
/** issue #199: chat wire format for llmEndpoint — 'openai' (default) or 'ollama'. */
|
|
80
|
+
llmProvider?: ChatProviderId;
|
|
81
|
+
llmFn?: (prompt: string) => Promise<string>;
|
|
82
|
+
llmRequestFn?: (req: LlmStructuredRequest) => Promise<string>;
|
|
83
|
+
/** issue #199: incremental render hook — fires per token chunk while the LLM streams. */
|
|
84
|
+
onDelta?: (delta: string) => void;
|
|
85
|
+
/** issue #199: injection seam for tests/serve; built from llmEndpoint when absent. */
|
|
86
|
+
llmChat?: ResolvedChat;
|
|
87
|
+
systemPrompt?: string;
|
|
88
|
+
embedFn?: any;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export interface ChatTurnOptions {
|
|
92
|
+
session?: ChatSession;
|
|
93
|
+
query: string;
|
|
94
|
+
loaded?: LoadedIndex;
|
|
95
|
+
indexDir?: string;
|
|
96
|
+
cacheDir?: string;
|
|
97
|
+
db?: string;
|
|
98
|
+
k?: number;
|
|
99
|
+
offline?: boolean;
|
|
30
100
|
llmEndpoint?: string;
|
|
31
101
|
llmModel?: string;
|
|
102
|
+
/** issue #199: chat wire format for llmEndpoint — 'openai' (default) or 'ollama'. */
|
|
103
|
+
llmProvider?: ChatProviderId;
|
|
32
104
|
llmFn?: (prompt: string) => Promise<string>;
|
|
105
|
+
llmRequestFn?: (req: LlmStructuredRequest) => Promise<string>;
|
|
106
|
+
/** issue #199: incremental render hook — fires per token chunk while the LLM streams. */
|
|
107
|
+
onDelta?: (delta: string) => void;
|
|
108
|
+
/** issue #199: injection seam for tests/serve; built from llmEndpoint when absent. */
|
|
109
|
+
llmChat?: ResolvedChat;
|
|
33
110
|
systemPrompt?: string;
|
|
34
111
|
embedFn?: any;
|
|
112
|
+
log?: (msg: string) => void;
|
|
35
113
|
}
|
|
36
114
|
|
|
37
115
|
export const DEFAULT_RAG_SYSTEM_PROMPT =
|
|
@@ -39,6 +117,108 @@ export const DEFAULT_RAG_SYSTEM_PROMPT =
|
|
|
39
117
|
'Always cite your sources using [file#heading] format. ' +
|
|
40
118
|
'If the context does not contain the answer, say that the notes do not have enough information.';
|
|
41
119
|
|
|
120
|
+
/**
|
|
121
|
+
* Returns the directory where chat sessions are persisted.
|
|
122
|
+
*/
|
|
123
|
+
export function getSessionsDir(indexDir: string): string {
|
|
124
|
+
return path.join(indexDir, 'sessions');
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const SESSION_ID_PATTERN = /^[a-zA-Z0-9_-]{1,64}$/;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Validates a session id before it is used as a filesystem path component.
|
|
131
|
+
*/
|
|
132
|
+
export function assertSafeSessionId(id: string): string {
|
|
133
|
+
if (!SESSION_ID_PATTERN.test(id)) {
|
|
134
|
+
throw new Error('Invalid session ID: expected 1-64 letters, numbers, hyphens, or underscores');
|
|
135
|
+
}
|
|
136
|
+
return id;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Atomically saves a ChatSession to disk (<indexDir>/sessions/<id>.json).
|
|
141
|
+
*/
|
|
142
|
+
export function saveChatSession(indexDir: string, session: ChatSession): string {
|
|
143
|
+
assertSafeSessionId(session.id);
|
|
144
|
+
const dir = getSessionsDir(indexDir);
|
|
145
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
146
|
+
const targetPath = path.join(dir, `${session.id}.json`);
|
|
147
|
+
const tmpPath = path.join(dir, `${session.id}.${Date.now()}.tmp`);
|
|
148
|
+
fs.writeFileSync(tmpPath, JSON.stringify(session, null, 2), 'utf8');
|
|
149
|
+
fs.renameSync(tmpPath, targetPath);
|
|
150
|
+
return targetPath;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Loads a ChatSession by id or creates a new one. Degrades safely if corrupt.
|
|
155
|
+
*/
|
|
156
|
+
export function loadChatSession(
|
|
157
|
+
indexDir: string,
|
|
158
|
+
id: string,
|
|
159
|
+
log: (msg: string) => void = () => {}
|
|
160
|
+
): ChatSession {
|
|
161
|
+
assertSafeSessionId(id);
|
|
162
|
+
const sessionPath = path.join(getSessionsDir(indexDir), `${id}.json`);
|
|
163
|
+
if (fs.existsSync(sessionPath)) {
|
|
164
|
+
try {
|
|
165
|
+
const raw = fs.readFileSync(sessionPath, 'utf8');
|
|
166
|
+
const parsed = JSON.parse(raw);
|
|
167
|
+
if (parsed && typeof parsed.id === 'string' && Array.isArray(parsed.turns)) {
|
|
168
|
+
return parsed as ChatSession;
|
|
169
|
+
}
|
|
170
|
+
log(`warning: session ${id} is malformed; starting a fresh session.`);
|
|
171
|
+
} catch (e: any) {
|
|
172
|
+
log(`warning: failed to parse session ${id} (${e.message}); starting fresh.`);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const now = new Date().toISOString();
|
|
177
|
+
return {
|
|
178
|
+
id,
|
|
179
|
+
createdAt: now,
|
|
180
|
+
updatedAt: now,
|
|
181
|
+
turns: [],
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Lists all existing ChatSessions in the indexDir, sorted by updatedAt descending.
|
|
187
|
+
*/
|
|
188
|
+
export function listChatSessions(
|
|
189
|
+
indexDir: string
|
|
190
|
+
): Array<{ id: string; createdAt: string; updatedAt: string; turnsCount: number }> {
|
|
191
|
+
const dir = getSessionsDir(indexDir);
|
|
192
|
+
if (!fs.existsSync(dir)) return [];
|
|
193
|
+
|
|
194
|
+
const out: Array<{ id: string; createdAt: string; updatedAt: string; turnsCount: number }> = [];
|
|
195
|
+
let entries: string[];
|
|
196
|
+
try {
|
|
197
|
+
entries = fs.readdirSync(dir);
|
|
198
|
+
} catch {
|
|
199
|
+
return [];
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
for (const file of entries) {
|
|
203
|
+
if (!file.endsWith('.json')) continue;
|
|
204
|
+
const p = path.join(dir, file);
|
|
205
|
+
try {
|
|
206
|
+
const raw = fs.readFileSync(p, 'utf8');
|
|
207
|
+
const s = JSON.parse(raw);
|
|
208
|
+
if (s && typeof s.id === 'string' && Array.isArray(s.turns)) {
|
|
209
|
+
out.push({
|
|
210
|
+
id: s.id,
|
|
211
|
+
createdAt: s.createdAt || s.updatedAt || '',
|
|
212
|
+
updatedAt: s.updatedAt || s.createdAt || '',
|
|
213
|
+
turnsCount: s.turns.length,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
} catch {}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
return out.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
|
|
220
|
+
}
|
|
221
|
+
|
|
42
222
|
/**
|
|
43
223
|
* Built-in extractive answer synthesizer for fully offline, zero-dependency QA.
|
|
44
224
|
*/
|
|
@@ -88,6 +268,55 @@ export function extractAnswerFallback(query: string, passages: SearchResultHit[]
|
|
|
88
268
|
return `Based on your notes:\n\n${body}`;
|
|
89
269
|
}
|
|
90
270
|
|
|
271
|
+
const CITATION_MENTION_RE = /\[([^\]\n]{1,200})\]/g;
|
|
272
|
+
|
|
273
|
+
/** `[file]` | `[file#heading]` | `[file › heading]` → {file, heading?}; null for non-citations. */
|
|
274
|
+
function parseCitationMention(text: string): { file: string; heading?: string } | null {
|
|
275
|
+
const t = text.trim();
|
|
276
|
+
if (!t) return null;
|
|
277
|
+
let file: string;
|
|
278
|
+
let heading: string | undefined;
|
|
279
|
+
const hash = t.indexOf('#');
|
|
280
|
+
const sep = t.indexOf('›');
|
|
281
|
+
if (hash > 0) {
|
|
282
|
+
file = t.slice(0, hash).trim();
|
|
283
|
+
heading = t.slice(hash + 1).trim();
|
|
284
|
+
} else if (sep > 0) {
|
|
285
|
+
file = t.slice(0, sep).trim();
|
|
286
|
+
heading = t.slice(sep + 1).trim();
|
|
287
|
+
} else {
|
|
288
|
+
file = t;
|
|
289
|
+
}
|
|
290
|
+
// markdown links, footnote refs, and empty brackets are not citations
|
|
291
|
+
if (!file || /https?:\/\//i.test(file)) return null;
|
|
292
|
+
return { file, heading: heading || undefined };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* issue #199 follow-up: which of the retrieved citations the answer text
|
|
297
|
+
* actually cites. Bracket-only scan (the exact formats the system prompt and
|
|
298
|
+
* the extractive fallback emit), exact file/heading match against the
|
|
299
|
+
* retrieval manifest, order of first mention. No mention → [].
|
|
300
|
+
*/
|
|
301
|
+
export function extractUsedCitations(answer: string, citations: Citation[]): Citation[] {
|
|
302
|
+
const used: Citation[] = [];
|
|
303
|
+
const seen = new Set<string>();
|
|
304
|
+
for (const m of answer.matchAll(CITATION_MENTION_RE)) {
|
|
305
|
+
const mention = parseCitationMention(m[1]);
|
|
306
|
+
if (!mention) continue;
|
|
307
|
+
const hit = citations.find(
|
|
308
|
+
(c) => c.file === mention.file && (mention.heading === undefined || c.heading === mention.heading)
|
|
309
|
+
);
|
|
310
|
+
if (!hit) continue;
|
|
311
|
+
const key = `${hit.file}\u0000${hit.heading ?? ''}`;
|
|
312
|
+
if (!seen.has(key)) {
|
|
313
|
+
seen.add(key);
|
|
314
|
+
used.push(hit);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return used;
|
|
318
|
+
}
|
|
319
|
+
|
|
91
320
|
/**
|
|
92
321
|
* Synthesizes an answer from retrieved passages using an LLM or fallback.
|
|
93
322
|
*/
|
|
@@ -96,9 +325,18 @@ export async function synthesizeAnswer(
|
|
|
96
325
|
passages: SearchResultHit[],
|
|
97
326
|
options: {
|
|
98
327
|
llmFn?: (prompt: string) => Promise<string>;
|
|
328
|
+
llmRequestFn?: (req: LlmStructuredRequest) => Promise<string>;
|
|
99
329
|
llmEndpoint?: string;
|
|
100
330
|
llmModel?: string;
|
|
331
|
+
/** issue #199: chat wire format for llmEndpoint — 'openai' (default) or 'ollama'. */
|
|
332
|
+
llmProvider?: ChatProviderId;
|
|
333
|
+
llmChat?: ResolvedChat;
|
|
334
|
+
/** issue #199: incremental render hook — fires per token chunk while the LLM streams. */
|
|
335
|
+
onDelta?: (delta: string) => void;
|
|
336
|
+
/** Cooperative cancel (serve /ask: client disconnected mid-stream). */
|
|
337
|
+
signal?: AbortSignal;
|
|
101
338
|
systemPrompt?: string;
|
|
339
|
+
conversationHistory?: string;
|
|
102
340
|
} = {}
|
|
103
341
|
): Promise<SynthesisResult> {
|
|
104
342
|
const citations: Citation[] = passages.map((p) => ({
|
|
@@ -107,46 +345,73 @@ export async function synthesizeAnswer(
|
|
|
107
345
|
score: p.score,
|
|
108
346
|
}));
|
|
109
347
|
|
|
110
|
-
|
|
111
|
-
|
|
348
|
+
// One prompt assembly for every stage: `system + user` reproduces the
|
|
349
|
+
// historical single-string prompt byte-for-byte; the split serves the
|
|
350
|
+
// structured transports (MCP sampling, issue #197).
|
|
351
|
+
const system = options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT;
|
|
352
|
+
const historyHeader = options.conversationHistory ? `\nConversation History:\n${options.conversationHistory}\n` : '';
|
|
353
|
+
const contextBlocks = passages
|
|
354
|
+
.map((p, i) => {
|
|
112
355
|
const header = p.heading ? `${p.file} › ${p.heading}` : p.file;
|
|
113
356
|
return `[Source ${i + 1}: ${header}]\n${p.snippet}`;
|
|
114
|
-
})
|
|
357
|
+
})
|
|
358
|
+
.join('\n\n---\n\n');
|
|
359
|
+
const user = `${historyHeader}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
|
|
360
|
+
|
|
361
|
+
// every stage returns through one shape so citationsUsed stays consistent
|
|
362
|
+
const finish = (answer: string, mode: SynthesisResult['mode']): SynthesisResult => ({
|
|
363
|
+
answer,
|
|
364
|
+
citations,
|
|
365
|
+
citationsUsed: extractUsedCitations(answer, citations),
|
|
366
|
+
passages,
|
|
367
|
+
mode,
|
|
368
|
+
});
|
|
115
369
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
370
|
+
// issue #197: structured hook (MCP sampling). Best-effort like the endpoint
|
|
371
|
+
// path — a refusal or timeout falls through to the extractive answer.
|
|
372
|
+
if (options.llmRequestFn) {
|
|
373
|
+
try {
|
|
374
|
+
const answer = await options.llmRequestFn({ system, user });
|
|
375
|
+
if (answer) return finish(answer, 'llm');
|
|
376
|
+
} catch {
|
|
377
|
+
// Fall through to extractive.
|
|
378
|
+
}
|
|
119
379
|
}
|
|
120
380
|
|
|
121
|
-
if (options.
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
}).join('\n\n---\n\n');
|
|
381
|
+
if (options.llmFn) {
|
|
382
|
+
const answer = await options.llmFn(system + user);
|
|
383
|
+
return finish(answer, 'llm');
|
|
384
|
+
}
|
|
126
385
|
|
|
127
|
-
|
|
386
|
+
// issue #199: unified chat provider path. `/chat/completions` by default
|
|
387
|
+
// (same wire format HyDE uses — llama.cpp, LM Studio, vLLM, Ollama /v1 all
|
|
388
|
+
// speak it), Ollama-native `/api/generate` under `--llm-provider ollama`.
|
|
389
|
+
// Streaming deltas flow to onDelta; the full text is assembled for the
|
|
390
|
+
// returned result. Best-effort like the old endpoint path — a refusal or
|
|
391
|
+
// idle timeout falls through to the extractive answer.
|
|
392
|
+
if (options.llmChat || options.llmEndpoint) {
|
|
393
|
+
const chat =
|
|
394
|
+
options.llmChat ??
|
|
395
|
+
resolveChatProvider({
|
|
396
|
+
provider: options.llmProvider ?? 'openai',
|
|
397
|
+
model: options.llmModel,
|
|
398
|
+
baseUrl: options.llmEndpoint,
|
|
399
|
+
});
|
|
128
400
|
try {
|
|
129
|
-
const
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
prompt,
|
|
135
|
-
stream: false,
|
|
136
|
-
}),
|
|
401
|
+
const answer = await chat.complete({
|
|
402
|
+
system,
|
|
403
|
+
user,
|
|
404
|
+
...(options.onDelta ? { onDelta: options.onDelta } : {}),
|
|
405
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
137
406
|
});
|
|
138
|
-
if (
|
|
139
|
-
const data: any = await resp.json();
|
|
140
|
-
const answer = data.response || data.text || '';
|
|
141
|
-
if (answer) return { answer, citations, passages };
|
|
142
|
-
}
|
|
407
|
+
if (answer) return finish(answer, 'endpoint');
|
|
143
408
|
} catch {
|
|
144
|
-
//
|
|
409
|
+
// Fall through to extractive.
|
|
145
410
|
}
|
|
146
411
|
}
|
|
147
412
|
|
|
148
413
|
const answer = extractAnswerFallback(query, passages);
|
|
149
|
-
return
|
|
414
|
+
return finish(answer, 'extractive');
|
|
150
415
|
}
|
|
151
416
|
|
|
152
417
|
/**
|
|
@@ -160,8 +425,12 @@ export async function askQuestion(opts: AskOptions): Promise<SynthesisResult> {
|
|
|
160
425
|
cacheDir = '',
|
|
161
426
|
k = 5,
|
|
162
427
|
llmFn,
|
|
428
|
+
llmRequestFn,
|
|
163
429
|
llmEndpoint,
|
|
164
430
|
llmModel,
|
|
431
|
+
llmProvider,
|
|
432
|
+
llmChat,
|
|
433
|
+
onDelta,
|
|
165
434
|
systemPrompt,
|
|
166
435
|
embedFn,
|
|
167
436
|
} = opts;
|
|
@@ -178,12 +447,160 @@ export async function askQuestion(opts: AskOptions): Promise<SynthesisResult> {
|
|
|
178
447
|
query,
|
|
179
448
|
k,
|
|
180
449
|
embedFn,
|
|
450
|
+
offline: opts.offline,
|
|
181
451
|
});
|
|
182
452
|
|
|
183
453
|
return synthesizeAnswer(query, passages, {
|
|
184
454
|
llmFn,
|
|
455
|
+
llmRequestFn,
|
|
185
456
|
llmEndpoint,
|
|
186
457
|
llmModel,
|
|
458
|
+
llmProvider,
|
|
459
|
+
llmChat,
|
|
460
|
+
onDelta,
|
|
187
461
|
systemPrompt,
|
|
188
462
|
});
|
|
189
463
|
}
|
|
464
|
+
|
|
465
|
+
/**
|
|
466
|
+
* Executes a single conversational turn in a ChatSession with context carry-over and source manifest.
|
|
467
|
+
*/
|
|
468
|
+
export async function chatTurn(opts: ChatTurnOptions): Promise<{
|
|
469
|
+
session: ChatSession;
|
|
470
|
+
turn: ChatTurn;
|
|
471
|
+
answer: string;
|
|
472
|
+
citations: Citation[];
|
|
473
|
+
manifest: ChatSourceManifestItem[];
|
|
474
|
+
/** Which synthesis stage produced the answer ('llm'/'endpoint'/'extractive'). */
|
|
475
|
+
mode: SynthesisResult['mode'];
|
|
476
|
+
/** Citations the answer text actually mentions (issue #199 follow-up). */
|
|
477
|
+
citationsUsed: Citation[];
|
|
478
|
+
}> {
|
|
479
|
+
const {
|
|
480
|
+
query,
|
|
481
|
+
loaded: inputLoaded,
|
|
482
|
+
indexDir,
|
|
483
|
+
cacheDir = '',
|
|
484
|
+
k = 5,
|
|
485
|
+
llmFn,
|
|
486
|
+
llmRequestFn,
|
|
487
|
+
llmEndpoint,
|
|
488
|
+
llmModel,
|
|
489
|
+
llmProvider,
|
|
490
|
+
llmChat,
|
|
491
|
+
onDelta,
|
|
492
|
+
systemPrompt,
|
|
493
|
+
embedFn,
|
|
494
|
+
log = () => {},
|
|
495
|
+
} = opts;
|
|
496
|
+
|
|
497
|
+
let loaded = inputLoaded;
|
|
498
|
+
if (!loaded) {
|
|
499
|
+
if (!indexDir) throw new Error('chatTurn requires either loaded index or indexDir');
|
|
500
|
+
loaded = loadIndex(indexDir);
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
let session = opts.session;
|
|
504
|
+
if (!session) {
|
|
505
|
+
const newId = crypto.randomUUID().slice(0, 8);
|
|
506
|
+
session = indexDir ? loadChatSession(indexDir, newId, log) : {
|
|
507
|
+
id: newId,
|
|
508
|
+
createdAt: new Date().toISOString(),
|
|
509
|
+
updatedAt: new Date().toISOString(),
|
|
510
|
+
turns: [],
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
// 1. Build contextual retrieval query for follow-up questions
|
|
515
|
+
let retrievalQuery = query;
|
|
516
|
+
if (session.turns.length > 0) {
|
|
517
|
+
const prevTurn = session.turns[session.turns.length - 1];
|
|
518
|
+
const prevTokens = tokenize(prevTurn.query).filter((t) => t.length > 3).slice(0, 4);
|
|
519
|
+
if (prevTokens.length > 0) {
|
|
520
|
+
retrievalQuery = `${query} ${prevTokens.join(' ')}`.trim();
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// 2. Search index
|
|
525
|
+
const passages = await searchIndex({
|
|
526
|
+
loaded,
|
|
527
|
+
cacheDir,
|
|
528
|
+
query: retrievalQuery,
|
|
529
|
+
k,
|
|
530
|
+
embedFn,
|
|
531
|
+
offline: opts.offline,
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
// 3. Construct source manifest from retrieved chunks
|
|
535
|
+
const manifest: ChatSourceManifestItem[] = passages.map((p) => {
|
|
536
|
+
const chunk = loaded.index?.chunks?.find(
|
|
537
|
+
(c: any) => c.file === p.file && (c.heading === p.heading || (!c.heading && !p.heading))
|
|
538
|
+
);
|
|
539
|
+
return {
|
|
540
|
+
file: p.file,
|
|
541
|
+
heading: p.heading,
|
|
542
|
+
headingPath: chunk?.headingPath,
|
|
543
|
+
startLine: chunk?.startLine,
|
|
544
|
+
endLine: chunk?.endLine,
|
|
545
|
+
chunkHash: chunk?.chunkHash,
|
|
546
|
+
score: p.score,
|
|
547
|
+
};
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
// 4. Conversation history string for LLM prompt
|
|
551
|
+
let conversationHistory = '';
|
|
552
|
+
if (session.turns.length > 0) {
|
|
553
|
+
conversationHistory = session.turns
|
|
554
|
+
.slice(-3)
|
|
555
|
+
.map((t) => `User: ${t.query}\nAssistant: ${t.answer}`)
|
|
556
|
+
.join('\n\n');
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// 5. Synthesize answer
|
|
560
|
+
const synth = await synthesizeAnswer(query, passages, {
|
|
561
|
+
llmFn,
|
|
562
|
+
llmRequestFn,
|
|
563
|
+
llmEndpoint,
|
|
564
|
+
llmModel,
|
|
565
|
+
llmProvider,
|
|
566
|
+
llmChat,
|
|
567
|
+
onDelta,
|
|
568
|
+
systemPrompt,
|
|
569
|
+
conversationHistory,
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
// 6. Append turn to session
|
|
573
|
+
const turn: ChatTurn = {
|
|
574
|
+
id: crypto.randomUUID(),
|
|
575
|
+
timestamp: new Date().toISOString(),
|
|
576
|
+
query,
|
|
577
|
+
answer: synth.answer,
|
|
578
|
+
citations: synth.citations,
|
|
579
|
+
manifest,
|
|
580
|
+
mode: synth.mode,
|
|
581
|
+
};
|
|
582
|
+
|
|
583
|
+
session.turns.push(turn);
|
|
584
|
+
session.updatedAt = turn.timestamp;
|
|
585
|
+
if (loaded.index?.db) session.db = loaded.index.db;
|
|
586
|
+
if (loaded.model?.id) session.model = loaded.model.id;
|
|
587
|
+
|
|
588
|
+
if (indexDir) {
|
|
589
|
+
try {
|
|
590
|
+
saveChatSession(indexDir, session);
|
|
591
|
+
} catch (e: any) {
|
|
592
|
+
log(`warning: failed to save chat session ${session.id}: ${e.message}`);
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
return {
|
|
597
|
+
session,
|
|
598
|
+
turn,
|
|
599
|
+
answer: turn.answer,
|
|
600
|
+
citations: turn.citations,
|
|
601
|
+
manifest: turn.manifest,
|
|
602
|
+
mode: turn.mode ?? 'extractive',
|
|
603
|
+
citationsUsed: extractUsedCitations(turn.answer, turn.citations),
|
|
604
|
+
};
|
|
605
|
+
}
|
|
606
|
+
|
package/src/repair.ts
CHANGED
|
@@ -20,6 +20,7 @@ export interface RepairAction {
|
|
|
20
20
|
| 'remove-stale-lock'
|
|
21
21
|
| 'remove-broken-vectors-bin'
|
|
22
22
|
| 'remove-corrupt-ivf'
|
|
23
|
+
| 'remove-corrupt-hnsw'
|
|
23
24
|
| 'rebuild-index'
|
|
24
25
|
| 'rewrite-hashes';
|
|
25
26
|
detail: string;
|
|
@@ -130,7 +131,7 @@ export function planRepairs(indexDir: string, opts: ProbeOptions = {}): RepairPl
|
|
|
130
131
|
}
|
|
131
132
|
}
|
|
132
133
|
|
|
133
|
-
// --- 3. corrupt
|
|
134
|
+
// --- 3. corrupt ANN sidecars (optional; loader silently ignores them) ---
|
|
134
135
|
const ivfPath = path.join(indexDir, 'ivf.json');
|
|
135
136
|
const ivfRaw = readJsonIfParsable(ivfPath);
|
|
136
137
|
if (ivfRaw.ok === false) {
|
|
@@ -139,12 +140,22 @@ export function planRepairs(indexDir: string, opts: ProbeOptions = {}): RepairPl
|
|
|
139
140
|
detail: 'ivf.json is not valid JSON — removing it re-enables brute-force ANN fallback',
|
|
140
141
|
});
|
|
141
142
|
}
|
|
143
|
+
// issue #201: hnsw.json gets the same corrupt-sidecar treatment
|
|
144
|
+
const hnswPath = path.join(indexDir, 'hnsw.json');
|
|
145
|
+
const hnswRaw = readJsonIfParsable(hnswPath);
|
|
146
|
+
if (hnswRaw.ok === false) {
|
|
147
|
+
plan.actions.push({
|
|
148
|
+
action: 'remove-corrupt-hnsw',
|
|
149
|
+
detail: 'hnsw.json is not valid JSON — removing it re-enables brute-force ANN fallback',
|
|
150
|
+
});
|
|
151
|
+
}
|
|
142
152
|
|
|
143
153
|
// --- 4. corrupt vectors.json ---
|
|
144
154
|
const liveWriter = plan.skipped.some(
|
|
145
155
|
(s) => s.action === 'remove-stale-lock' && /live writer/.test(s.detail),
|
|
146
156
|
);
|
|
147
|
-
const
|
|
157
|
+
const vectorsPath = path.join(indexDir, 'vectors.json');
|
|
158
|
+
const vectorsRaw = readJsonIfParsable(vectorsPath);
|
|
148
159
|
if (vectorsRaw.ok === false) {
|
|
149
160
|
if (db && !liveWriter) {
|
|
150
161
|
plan.actions.push({ action: 'rebuild-index', detail: `vectors.json is not valid JSON — rebuild from ${db}` });
|
|
@@ -161,6 +172,41 @@ export function planRepairs(indexDir: string, opts: ProbeOptions = {}): RepairPl
|
|
|
161
172
|
}
|
|
162
173
|
}
|
|
163
174
|
|
|
175
|
+
// --- 4b. vectors.json SHA-256 sidecar mismatch (issue #166) ---
|
|
176
|
+
// `mdss check` reports this as an integrity failure; without a planned
|
|
177
|
+
// repair `check --fix` printed "nothing to repair" and left the index
|
|
178
|
+
// broken. Mirror the check's hashing (raw bytes vs first sidecar token)
|
|
179
|
+
// and plan the same rebuild as the corrupt-JSON path — rewriting the
|
|
180
|
+
// sidecar would bless whatever tampered content is on disk.
|
|
181
|
+
if (vectorsRaw.ok === true) {
|
|
182
|
+
const vectorsShaPath = `${vectorsPath}.sha256`;
|
|
183
|
+
if (fs.existsSync(vectorsShaPath)) {
|
|
184
|
+
let mismatch: string | null = null;
|
|
185
|
+
try {
|
|
186
|
+
const actual = crypto.createHash('sha256').update(fs.readFileSync(vectorsPath)).digest('hex');
|
|
187
|
+
const expected = fs.readFileSync(vectorsShaPath, 'utf8').trim().split(/\s+/)[0];
|
|
188
|
+
if (expected && expected !== actual) mismatch = 'SHA-256 mismatch';
|
|
189
|
+
} catch (e: any) {
|
|
190
|
+
mismatch = e.message;
|
|
191
|
+
}
|
|
192
|
+
if (mismatch) {
|
|
193
|
+
if (db && !liveWriter) {
|
|
194
|
+
plan.actions.push({ action: 'rebuild-index', detail: `vectors.json integrity check failed (${mismatch}) — rebuild from ${db}` });
|
|
195
|
+
} else if (liveWriter) {
|
|
196
|
+
plan.skipped.push({
|
|
197
|
+
action: 'rebuild-index',
|
|
198
|
+
detail: `vectors.json integrity check failed (${mismatch}), but a live writer holds the lock — retry after it finishes`,
|
|
199
|
+
});
|
|
200
|
+
} else {
|
|
201
|
+
plan.skipped.push({
|
|
202
|
+
action: 'rebuild-index',
|
|
203
|
+
detail: `vectors.json integrity check failed (${mismatch}) — pass --db so --fix can rebuild it`,
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
164
210
|
// --- 5. damaged .hashes.json ---
|
|
165
211
|
const hashesRaw = readJsonIfParsable(path.join(indexDir, '.hashes.json'));
|
|
166
212
|
if (hashesRaw.ok === false) {
|
|
@@ -213,6 +259,9 @@ export function applyRepairs(
|
|
|
213
259
|
case 'remove-corrupt-ivf':
|
|
214
260
|
fs.rmSync(path.join(indexDir, 'ivf.json'), { force: true });
|
|
215
261
|
break;
|
|
262
|
+
case 'remove-corrupt-hnsw':
|
|
263
|
+
fs.rmSync(path.join(indexDir, 'hnsw.json'), { force: true });
|
|
264
|
+
break;
|
|
216
265
|
case 'rewrite-hashes': {
|
|
217
266
|
const hashes = recomputeHashes(opts.db!);
|
|
218
267
|
fs.writeFileSync(path.join(indexDir, '.hashes.json'), JSON.stringify(hashes, null, 2));
|