md-semantic-search 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +198 -8
  2. package/RESEARCH.md +25 -3
  3. package/bin/cli.mjs +657 -29
  4. package/dist/completions.d.ts +25 -0
  5. package/dist/completions.js +238 -0
  6. package/dist/core.d.ts +31 -0
  7. package/dist/core.js +121 -1
  8. package/dist/download-progress.d.ts +52 -0
  9. package/dist/download-progress.js +112 -0
  10. package/dist/index-format.js +8 -1
  11. package/dist/index.d.ts +6 -4
  12. package/dist/index.js +5 -4
  13. package/dist/indexer.d.ts +1 -0
  14. package/dist/indexer.js +5 -5
  15. package/dist/ivf.d.ts +14 -7
  16. package/dist/ivf.js +126 -18
  17. package/dist/mcp.d.ts +143 -4
  18. package/dist/mcp.js +499 -23
  19. package/dist/models.d.ts +1 -1
  20. package/dist/models.js +52 -31
  21. package/dist/open.d.ts +43 -0
  22. package/dist/open.js +84 -0
  23. package/dist/providers.d.ts +21 -0
  24. package/dist/providers.js +104 -0
  25. package/dist/query-cache-disk.d.ts +14 -0
  26. package/dist/query-cache-disk.js +175 -0
  27. package/dist/rag.d.ts +72 -7
  28. package/dist/rag.js +191 -13
  29. package/dist/repair.d.ts +36 -0
  30. package/dist/repair.js +208 -0
  31. package/dist/rerank.d.ts +20 -4
  32. package/dist/rerank.js +14 -4
  33. package/dist/search.d.ts +35 -0
  34. package/dist/search.js +175 -5
  35. package/dist/serve.d.ts +126 -0
  36. package/dist/serve.js +676 -46
  37. package/dist/tui.d.ts +8 -0
  38. package/dist/tui.js +80 -13
  39. package/dist/watcher.d.ts +16 -0
  40. package/dist/watcher.js +41 -0
  41. package/dist/webui.d.ts +13 -0
  42. package/dist/webui.js +252 -0
  43. package/dist/wikilinks.d.ts +31 -0
  44. package/dist/wikilinks.js +185 -1
  45. package/package.json +1 -1
  46. package/src/completions.ts +255 -0
  47. package/src/core.ts +160 -1
  48. package/src/download-progress.ts +146 -0
  49. package/src/index-format.ts +7 -1
  50. package/src/index.ts +11 -2
  51. package/src/indexer.ts +8 -4
  52. package/src/ivf.ts +157 -20
  53. package/src/mcp.ts +514 -23
  54. package/src/models.ts +58 -33
  55. package/src/open.ts +119 -0
  56. package/src/providers.ts +134 -0
  57. package/src/query-cache-disk.ts +191 -0
  58. package/src/rag.ts +279 -17
  59. package/src/repair.ts +245 -0
  60. package/src/rerank.ts +43 -8
  61. package/src/search.ts +199 -6
  62. package/src/serve.ts +726 -43
  63. package/src/tui.ts +86 -14
  64. package/src/watcher.ts +46 -0
  65. package/src/webui.ts +255 -0
  66. package/src/wikilinks.ts +220 -1
package/dist/rag.js CHANGED
@@ -1,15 +1,90 @@
1
- /**
2
- * Offline QA & Local RAG Answer Synthesis (issue #101).
3
- *
4
- * Implements grounded question answering with strict provenance citations ([file#heading]),
5
- * supporting local LLM bridges (Ollama / LocalAI / OpenAI-compatible endpoint) and a
6
- * fast built-in zero-dependency extractive synthesis fallback.
7
- */
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import crypto from 'node:crypto';
8
4
  import { searchIndex, loadIndex } from './search.js';
9
5
  import { tokenize } from './lexical.js';
10
6
  export const DEFAULT_RAG_SYSTEM_PROMPT = 'You are a helpful assistant answering questions using only the provided Markdown notes. ' +
11
7
  'Always cite your sources using [file#heading] format. ' +
12
8
  'If the context does not contain the answer, say that the notes do not have enough information.';
9
+ /**
10
+ * Returns the directory where chat sessions are persisted.
11
+ */
12
+ export function getSessionsDir(indexDir) {
13
+ return path.join(indexDir, 'sessions');
14
+ }
15
+ /**
16
+ * Atomically saves a ChatSession to disk (<indexDir>/sessions/<id>.json).
17
+ */
18
+ export function saveChatSession(indexDir, session) {
19
+ const dir = getSessionsDir(indexDir);
20
+ fs.mkdirSync(dir, { recursive: true });
21
+ const targetPath = path.join(dir, `${session.id}.json`);
22
+ const tmpPath = path.join(dir, `${session.id}.${Date.now()}.tmp`);
23
+ fs.writeFileSync(tmpPath, JSON.stringify(session, null, 2), 'utf8');
24
+ fs.renameSync(tmpPath, targetPath);
25
+ return targetPath;
26
+ }
27
+ /**
28
+ * Loads a ChatSession by id or creates a new one. Degrades safely if corrupt.
29
+ */
30
+ export function loadChatSession(indexDir, id, log = () => { }) {
31
+ const sessionPath = path.join(getSessionsDir(indexDir), `${id}.json`);
32
+ if (fs.existsSync(sessionPath)) {
33
+ try {
34
+ const raw = fs.readFileSync(sessionPath, 'utf8');
35
+ const parsed = JSON.parse(raw);
36
+ if (parsed && typeof parsed.id === 'string' && Array.isArray(parsed.turns)) {
37
+ return parsed;
38
+ }
39
+ log(`warning: session ${id} is malformed; starting a fresh session.`);
40
+ }
41
+ catch (e) {
42
+ log(`warning: failed to parse session ${id} (${e.message}); starting fresh.`);
43
+ }
44
+ }
45
+ const now = new Date().toISOString();
46
+ return {
47
+ id,
48
+ createdAt: now,
49
+ updatedAt: now,
50
+ turns: [],
51
+ };
52
+ }
53
+ /**
54
+ * Lists all existing ChatSessions in the indexDir, sorted by updatedAt descending.
55
+ */
56
+ export function listChatSessions(indexDir) {
57
+ const dir = getSessionsDir(indexDir);
58
+ if (!fs.existsSync(dir))
59
+ return [];
60
+ const out = [];
61
+ let entries;
62
+ try {
63
+ entries = fs.readdirSync(dir);
64
+ }
65
+ catch {
66
+ return [];
67
+ }
68
+ for (const file of entries) {
69
+ if (!file.endsWith('.json'))
70
+ continue;
71
+ const p = path.join(dir, file);
72
+ try {
73
+ const raw = fs.readFileSync(p, 'utf8');
74
+ const s = JSON.parse(raw);
75
+ if (s && typeof s.id === 'string' && Array.isArray(s.turns)) {
76
+ out.push({
77
+ id: s.id,
78
+ createdAt: s.createdAt || s.updatedAt || '',
79
+ updatedAt: s.updatedAt || s.createdAt || '',
80
+ turnsCount: s.turns.length,
81
+ });
82
+ }
83
+ }
84
+ catch { }
85
+ }
86
+ return out.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
87
+ }
13
88
  /**
14
89
  * Built-in extractive answer synthesizer for fully offline, zero-dependency QA.
15
90
  */
@@ -62,21 +137,26 @@ export async function synthesizeAnswer(query, passages, options = {}) {
62
137
  heading: p.heading,
63
138
  score: p.score,
64
139
  }));
140
+ const historyHeader = options.conversationHistory ? `\nConversation History:\n${options.conversationHistory}\n` : '';
65
141
  if (options.llmFn) {
66
- const contextBlocks = passages.map((p, i) => {
142
+ const contextBlocks = passages
143
+ .map((p, i) => {
67
144
  const header = p.heading ? `${p.file} › ${p.heading}` : p.file;
68
145
  return `[Source ${i + 1}: ${header}]\n${p.snippet}`;
69
- }).join('\n\n---\n\n');
70
- const prompt = `${options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
146
+ })
147
+ .join('\n\n---\n\n');
148
+ const prompt = `${options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT}${historyHeader}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
71
149
  const answer = await options.llmFn(prompt);
72
150
  return { answer, citations, passages };
73
151
  }
74
152
  if (options.llmEndpoint) {
75
- const contextBlocks = passages.map((p, i) => {
153
+ const contextBlocks = passages
154
+ .map((p, i) => {
76
155
  const header = p.heading ? `${p.file} › ${p.heading}` : p.file;
77
156
  return `[Source ${i + 1}: ${header}]\n${p.snippet}`;
78
- }).join('\n\n---\n\n');
79
- const prompt = `${options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
157
+ })
158
+ .join('\n\n---\n\n');
159
+ const prompt = `${options.systemPrompt || DEFAULT_RAG_SYSTEM_PROMPT}${historyHeader}\n\nContext:\n${contextBlocks}\n\nQuestion: ${query}\n\nAnswer:`;
80
160
  try {
81
161
  const resp = await fetch(`${options.llmEndpoint}/api/generate`, {
82
162
  method: 'POST',
@@ -126,3 +206,101 @@ export async function askQuestion(opts) {
126
206
  systemPrompt,
127
207
  });
128
208
  }
209
+ /**
210
+ * Executes a single conversational turn in a ChatSession with context carry-over and source manifest.
211
+ */
212
+ export async function chatTurn(opts) {
213
+ const { query, loaded: inputLoaded, indexDir, cacheDir = '', k = 5, llmFn, llmEndpoint, llmModel, systemPrompt, embedFn, log = () => { }, } = opts;
214
+ let loaded = inputLoaded;
215
+ if (!loaded) {
216
+ if (!indexDir)
217
+ throw new Error('chatTurn requires either loaded index or indexDir');
218
+ loaded = loadIndex(indexDir);
219
+ }
220
+ let session = opts.session;
221
+ if (!session) {
222
+ const newId = crypto.randomUUID().slice(0, 8);
223
+ session = indexDir ? loadChatSession(indexDir, newId, log) : {
224
+ id: newId,
225
+ createdAt: new Date().toISOString(),
226
+ updatedAt: new Date().toISOString(),
227
+ turns: [],
228
+ };
229
+ }
230
+ // 1. Build contextual retrieval query for follow-up questions
231
+ let retrievalQuery = query;
232
+ if (session.turns.length > 0) {
233
+ const prevTurn = session.turns[session.turns.length - 1];
234
+ const prevTokens = tokenize(prevTurn.query).filter((t) => t.length > 3).slice(0, 4);
235
+ if (prevTokens.length > 0) {
236
+ retrievalQuery = `${query} ${prevTokens.join(' ')}`.trim();
237
+ }
238
+ }
239
+ // 2. Search index
240
+ const passages = await searchIndex({
241
+ loaded,
242
+ cacheDir,
243
+ query: retrievalQuery,
244
+ k,
245
+ embedFn,
246
+ });
247
+ // 3. Construct source manifest from retrieved chunks
248
+ const manifest = passages.map((p) => {
249
+ const chunk = loaded.index?.chunks?.find((c) => c.file === p.file && (c.heading === p.heading || (!c.heading && !p.heading)));
250
+ return {
251
+ file: p.file,
252
+ heading: p.heading,
253
+ headingPath: chunk?.headingPath,
254
+ startLine: chunk?.startLine,
255
+ endLine: chunk?.endLine,
256
+ chunkHash: chunk?.chunkHash,
257
+ score: p.score,
258
+ };
259
+ });
260
+ // 4. Conversation history string for LLM prompt
261
+ let conversationHistory = '';
262
+ if (session.turns.length > 0) {
263
+ conversationHistory = session.turns
264
+ .slice(-3)
265
+ .map((t) => `User: ${t.query}\nAssistant: ${t.answer}`)
266
+ .join('\n\n');
267
+ }
268
+ // 5. Synthesize answer
269
+ const synth = await synthesizeAnswer(query, passages, {
270
+ llmFn,
271
+ llmEndpoint,
272
+ llmModel,
273
+ systemPrompt,
274
+ conversationHistory,
275
+ });
276
+ // 6. Append turn to session
277
+ const turn = {
278
+ id: crypto.randomUUID(),
279
+ timestamp: new Date().toISOString(),
280
+ query,
281
+ answer: synth.answer,
282
+ citations: synth.citations,
283
+ manifest,
284
+ };
285
+ session.turns.push(turn);
286
+ session.updatedAt = turn.timestamp;
287
+ if (loaded.index?.db)
288
+ session.db = loaded.index.db;
289
+ if (loaded.model?.id)
290
+ session.model = loaded.model.id;
291
+ if (indexDir) {
292
+ try {
293
+ saveChatSession(indexDir, session);
294
+ }
295
+ catch (e) {
296
+ log(`warning: failed to save chat session ${session.id}: ${e.message}`);
297
+ }
298
+ }
299
+ return {
300
+ session,
301
+ turn,
302
+ answer: turn.answer,
303
+ citations: turn.citations,
304
+ manifest: turn.manifest,
305
+ };
306
+ }
@@ -0,0 +1,36 @@
1
+ export interface RepairAction {
2
+ action: 'remove-stale-lock' | 'remove-broken-vectors-bin' | 'remove-corrupt-ivf' | 'rebuild-index' | 'rewrite-hashes';
3
+ detail: string;
4
+ }
5
+ export interface RepairPlan {
6
+ actions: RepairAction[];
7
+ skipped: RepairAction[];
8
+ }
9
+ export interface ApplyResult {
10
+ performed: RepairAction[];
11
+ failed: Array<RepairAction & {
12
+ error: string;
13
+ }>;
14
+ }
15
+ interface ProbeOptions {
16
+ db?: string | null;
17
+ log?: (msg: string) => void;
18
+ }
19
+ /** Recompute the {relativePath: md5} map exactly as the indexer stores it. */
20
+ export declare function recomputeHashes(db: string): Record<string, string>;
21
+ /**
22
+ * Probe the index directory and decide what CAN be safely repaired.
23
+ * Never mutates anything — safe to call for --dry-run.
24
+ */
25
+ export declare function planRepairs(indexDir: string, opts?: ProbeOptions): RepairPlan;
26
+ /** Execute planned actions in order. dryRun=true performs nothing. */
27
+ export declare function applyRepairs(indexDir: string, plan: RepairPlan, opts?: {
28
+ dryRun?: boolean;
29
+ db?: string | null;
30
+ cacheDir?: string;
31
+ modelName?: string;
32
+ embedFn?: any;
33
+ offline?: boolean;
34
+ log?: (msg: string) => void;
35
+ }): Promise<ApplyResult>;
36
+ export {};
package/dist/repair.js ADDED
@@ -0,0 +1,208 @@
1
+ /**
2
+ * Safe auto-repairs for `mdss check --fix` (issue #117).
3
+ *
4
+ * Plan/apply split: planRepairs() probes the index directory WITHOUT mutating
5
+ * anything (so --dry-run can print the exact plan), applyRepairs() executes
6
+ * actions in order and reports per-action outcomes. Source Markdown is NEVER
7
+ * touched; the only reads from --db are for recomputing `.hashes.json` or a
8
+ * full index rebuild.
9
+ */
10
+ import crypto from 'node:crypto';
11
+ import fs from 'node:fs';
12
+ import path from 'node:path';
13
+ import { LOCK_FILENAME, pidAlive, walkMarkdown } from './core.js';
14
+ import { DEFAULT_MODEL } from './models.js';
15
+ const LOCK_STALE_MS = 10 * 60 * 1000;
16
+ function readJsonIfParsable(filePath) {
17
+ let raw;
18
+ try {
19
+ raw = fs.readFileSync(filePath, 'utf8');
20
+ }
21
+ catch {
22
+ return { ok: 'missing' };
23
+ }
24
+ try {
25
+ return { ok: true, value: JSON.parse(raw) };
26
+ }
27
+ catch {
28
+ return { ok: false };
29
+ }
30
+ }
31
+ /** Recompute the {relativePath: md5} map exactly as the indexer stores it. */
32
+ export function recomputeHashes(db) {
33
+ const hashes = {};
34
+ for (const f of walkMarkdown(db)) {
35
+ const rel = path.relative(db, f).split(path.sep).join('/');
36
+ const md5 = crypto.createHash('md5').update(fs.readFileSync(f)).digest('hex');
37
+ hashes[rel] = md5;
38
+ }
39
+ return hashes;
40
+ }
41
+ /**
42
+ * Probe the index directory and decide what CAN be safely repaired.
43
+ * Never mutates anything — safe to call for --dry-run.
44
+ */
45
+ export function planRepairs(indexDir, opts = {}) {
46
+ const plan = { actions: [], skipped: [] };
47
+ const db = opts.db ?? null;
48
+ // --- 1. stale / abandoned lock ---
49
+ const lockPath = path.join(indexDir, LOCK_FILENAME);
50
+ if (fs.existsSync(lockPath)) {
51
+ let info = null;
52
+ try {
53
+ info = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
54
+ }
55
+ catch {
56
+ info = null;
57
+ }
58
+ const pid = Number.isInteger(info?.pid) ? info.pid : null;
59
+ let statAgeMs = Infinity;
60
+ try {
61
+ statAgeMs = Date.now() - fs.statSync(lockPath).mtimeMs;
62
+ }
63
+ catch {
64
+ /* vanished between exists and stat */
65
+ }
66
+ if (!info) {
67
+ plan.actions.push({ action: 'remove-stale-lock', detail: `${LOCK_FILENAME} is unreadable/abandoned` });
68
+ }
69
+ else if (pid !== null && !pidAlive(pid)) {
70
+ plan.actions.push({ action: 'remove-stale-lock', detail: `${LOCK_FILENAME} held by dead pid ${pid}` });
71
+ }
72
+ else if (statAgeMs > LOCK_STALE_MS) {
73
+ plan.actions.push({
74
+ action: 'remove-stale-lock',
75
+ detail: `${LOCK_FILENAME} is older than ${Math.round(LOCK_STALE_MS / 60000)} min`,
76
+ });
77
+ }
78
+ else {
79
+ plan.skipped.push({
80
+ action: 'remove-stale-lock',
81
+ detail: `live writer holds the lock${pid ? ` (pid ${pid})` : ''}`,
82
+ });
83
+ }
84
+ }
85
+ // --- 2. vectors.bin integrity (SHA-256 sidecar) ---
86
+ const binPath = path.join(indexDir, 'vectors.bin');
87
+ const shaPath = `${binPath}.sha256`;
88
+ if (fs.existsSync(binPath)) {
89
+ let broken = null;
90
+ try {
91
+ const actual = crypto.createHash('sha256').update(fs.readFileSync(binPath)).digest('hex');
92
+ if (!fs.existsSync(shaPath)) {
93
+ broken = 'missing vectors.bin.sha256 sidecar';
94
+ }
95
+ else {
96
+ const expected = fs.readFileSync(shaPath, 'utf8').trim().split(/\s+/)[0];
97
+ if (!expected || expected !== actual)
98
+ broken = 'SHA-256 mismatch';
99
+ }
100
+ }
101
+ catch (e) {
102
+ broken = e.message;
103
+ }
104
+ if (broken) {
105
+ plan.actions.push({
106
+ action: 'remove-broken-vectors-bin',
107
+ detail: `vectors.bin ignored (${broken}) — search falls back to vectors.json`,
108
+ });
109
+ }
110
+ }
111
+ // --- 3. corrupt ivf.json (optional ANN sidecar; loader silently ignores it) ---
112
+ const ivfPath = path.join(indexDir, 'ivf.json');
113
+ const ivfRaw = readJsonIfParsable(ivfPath);
114
+ if (ivfRaw.ok === false) {
115
+ plan.actions.push({
116
+ action: 'remove-corrupt-ivf',
117
+ detail: 'ivf.json is not valid JSON — removing it re-enables brute-force ANN fallback',
118
+ });
119
+ }
120
+ // --- 4. corrupt vectors.json ---
121
+ const liveWriter = plan.skipped.some((s) => s.action === 'remove-stale-lock' && /live writer/.test(s.detail));
122
+ const vectorsRaw = readJsonIfParsable(path.join(indexDir, 'vectors.json'));
123
+ if (vectorsRaw.ok === false) {
124
+ if (db && !liveWriter) {
125
+ plan.actions.push({ action: 'rebuild-index', detail: `vectors.json is not valid JSON — rebuild from ${db}` });
126
+ }
127
+ else if (liveWriter) {
128
+ plan.skipped.push({
129
+ action: 'rebuild-index',
130
+ detail: 'vectors.json is not valid JSON, but a live writer holds the lock — retry after it finishes',
131
+ });
132
+ }
133
+ else {
134
+ plan.skipped.push({
135
+ action: 'rebuild-index',
136
+ detail: 'vectors.json is not valid JSON — pass --db so --fix can rebuild it',
137
+ });
138
+ }
139
+ }
140
+ // --- 5. damaged .hashes.json ---
141
+ const hashesRaw = readJsonIfParsable(path.join(indexDir, '.hashes.json'));
142
+ if (hashesRaw.ok === false) {
143
+ if (db) {
144
+ plan.actions.push({ action: 'rewrite-hashes', detail: '.hashes.json is not valid JSON — recompute from source md5' });
145
+ }
146
+ else {
147
+ plan.skipped.push({
148
+ action: 'rewrite-hashes',
149
+ detail: '.hashes.json is not valid JSON — pass --db so --fix can recompute it',
150
+ });
151
+ }
152
+ }
153
+ return plan;
154
+ }
155
+ /** Execute planned actions in order. dryRun=true performs nothing. */
156
+ export function applyRepairs(indexDir, plan, opts = {}) {
157
+ const performed = [];
158
+ const failed = [];
159
+ const log = opts.log ?? (() => { });
160
+ const run = async () => {
161
+ for (const action of plan.actions) {
162
+ if (opts.dryRun) {
163
+ performed.push(action);
164
+ continue;
165
+ }
166
+ try {
167
+ switch (action.action) {
168
+ case 'remove-stale-lock':
169
+ fs.unlinkSync(path.join(indexDir, LOCK_FILENAME));
170
+ break;
171
+ case 'remove-broken-vectors-bin':
172
+ fs.rmSync(path.join(indexDir, 'vectors.bin'), { force: true });
173
+ fs.rmSync(`${path.join(indexDir, 'vectors.bin')}.sha256`, { force: true });
174
+ break;
175
+ case 'remove-corrupt-ivf':
176
+ fs.rmSync(path.join(indexDir, 'ivf.json'), { force: true });
177
+ break;
178
+ case 'rewrite-hashes': {
179
+ const hashes = recomputeHashes(opts.db);
180
+ fs.writeFileSync(path.join(indexDir, '.hashes.json'), JSON.stringify(hashes, null, 2));
181
+ break;
182
+ }
183
+ case 'rebuild-index': {
184
+ const { buildIndex } = await import('./indexer.js');
185
+ const rebuildOpts = {
186
+ db: opts.db,
187
+ indexDir,
188
+ cacheDir: opts.cacheDir,
189
+ modelName: opts.modelName || DEFAULT_MODEL,
190
+ log,
191
+ _lockHeld: true, // we may have JUST removed a stale lock; never double-lock inside fix
192
+ };
193
+ if (opts.embedFn)
194
+ rebuildOpts.embedFn = opts.embedFn; // test DI — keeps repairs model-free
195
+ await buildIndex(rebuildOpts);
196
+ break;
197
+ }
198
+ }
199
+ performed.push(action);
200
+ }
201
+ catch (e) {
202
+ failed.push({ ...action, error: e.message });
203
+ }
204
+ }
205
+ return { performed, failed };
206
+ };
207
+ return run();
208
+ }
package/dist/rerank.d.ts CHANGED
@@ -1,10 +1,26 @@
1
+ export interface RerankerTokenizer {
2
+ (queries: string[], options: {
3
+ text_pair: string[];
4
+ padding: boolean;
5
+ truncation: boolean;
6
+ }): unknown;
7
+ }
8
+ export interface RerankerModelOutput {
9
+ logits: {
10
+ tolist(): unknown;
11
+ };
12
+ }
13
+ export interface RerankerModel {
14
+ (inputs: unknown): Promise<RerankerModelOutput>;
15
+ }
16
+ export interface LoadedReranker {
17
+ tokenizer: RerankerTokenizer;
18
+ model: RerankerModel;
19
+ }
1
20
  export declare const RERANK_MODEL = "Xenova/bge-reranker-base";
2
21
  export declare function getReranker(cacheDir: string, offline?: boolean, retryOpts?: {
3
22
  maxRetries?: number;
4
23
  delays?: number[];
5
24
  log?: (msg: string) => void;
6
- }): Promise<{
7
- tokenizer: any;
8
- model: any;
9
- }>;
25
+ }): Promise<LoadedReranker>;
10
26
  export declare function rerankScores(query: string, passages: string[], cacheDir: string, offline?: boolean): Promise<number[]>;
package/dist/rerank.js CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Cross-encoder re-ranking (issue #15).
3
3
  */
4
- import { resolveSessionOptions, retryWithBackoff } from './core.js';
4
+ import { resolveSessionOptions, retryWithBackoff, emitDownloadEvent } from './core.js';
5
5
  const _rerankers = new Map();
6
6
  export const RERANK_MODEL = 'Xenova/bge-reranker-base';
7
7
  export async function getReranker(cacheDir, offline = false, retryOpts = {}) {
@@ -11,13 +11,14 @@ export async function getReranker(cacheDir, offline = false, retryOpts = {}) {
11
11
  if (_rerankers.has(key))
12
12
  return _rerankers.get(key);
13
13
  const loadPipeline = async () => {
14
- const { AutoModelForSequenceClassification, AutoTokenizer, env } = await import('@huggingface/transformers');
14
+ const { AutoModelForSequenceClassification, AutoTokenizer, env } = (await import('@huggingface/transformers'));
15
15
  if (cacheDir)
16
16
  env.cacheDir = cacheDir;
17
17
  env.allowRemoteModels = !offline;
18
- const tokenizer = await AutoTokenizer.from_pretrained(RERANK_MODEL);
18
+ const tokenizer = await AutoTokenizer.from_pretrained(RERANK_MODEL, { progress_callback: emitDownloadEvent });
19
19
  const loadOpts = {
20
20
  dtype: 'q8',
21
+ progress_callback: emitDownloadEvent,
21
22
  ...(sessionOptions ? { session_options: sessionOptions } : {}),
22
23
  };
23
24
  const model = await AutoModelForSequenceClassification.from_pretrained(RERANK_MODEL, loadOpts);
@@ -37,5 +38,14 @@ export async function rerankScores(query, passages, cacheDir, offline = false) {
37
38
  const queries = new Array(passages.length).fill(query);
38
39
  const inputs = tokenizer(queries, { text_pair: passages, padding: true, truncation: true });
39
40
  const outputs = await model(inputs);
40
- return outputs.logits.tolist().map((row) => row[0]);
41
+ const rawLogits = outputs?.logits?.tolist();
42
+ if (!Array.isArray(rawLogits)) {
43
+ throw new Error('reranker model returned invalid logits structure');
44
+ }
45
+ return rawLogits.map((row, idx) => {
46
+ if (!Array.isArray(row) || typeof row[0] !== 'number' || !Number.isFinite(row[0])) {
47
+ throw new Error(`reranker model returned non-finite score at index ${idx}`);
48
+ }
49
+ return row[0];
50
+ });
41
51
  }
package/dist/search.d.ts CHANGED
@@ -81,6 +81,28 @@ export declare function getRuntimeGraph(runtime: RuntimeIndexState): {
81
81
  graph: RelationshipGraph;
82
82
  pageRank: Map<string, number>;
83
83
  };
84
+ /**
85
+ * Issue #127 pure helpers — exported for deterministic unit tests.
86
+ */
87
+ /** Frontmatter `created` → `updated` → file mtime; null when nothing parses. */
88
+ export declare function resolvePassageAgeMs(meta: {
89
+ created?: unknown;
90
+ updated?: unknown;
91
+ } | undefined, absFile?: string): number | null;
92
+ /** Multiplicative recency boost: 0.5^(ageDays / halfLifeDays); 1 when off/unknown/future. */
93
+ export declare function recencyBoost(ageMs: number | null, halfLifeDays: number, now?: number): number;
94
+ /**
95
+ * HyDE passage generation via an OpenAI-compatible chat endpoint (issue #125).
96
+ * Returns null on ANY failure — HyDE must degrade to PRF/baseline silently.
97
+ * Injectable fetch for tests.
98
+ */
99
+ export declare function hydePassage(endpoint: string, query: string, model?: string, fetchImpl?: typeof fetch): Promise<string | null>;
100
+ /**
101
+ * Salient-term extraction for PRF (issue #125): tokens that appear in the
102
+ * top-m feedback passages, are not query terms or stopwords, and are ranked
103
+ * by (tf-in-feedback × distinct-passage coverage). Pure + deterministic.
104
+ */
105
+ export declare function extractExpansionTerms(topTexts: string[], queryTerms: string[], maxTerms: number): string[];
84
106
  export declare function loadIndex(indexDir: string): {
85
107
  index: IndexFile;
86
108
  model: ModelDescriptor;
@@ -113,10 +135,23 @@ export interface SearchOptions {
113
135
  maxPerFile?: number;
114
136
  maxPerDoc?: number;
115
137
  useQueryCache?: boolean;
138
+ /** issue #114: persist query embeddings to <cacheDir>/query-cache.json (default on). */
139
+ queryDiskCache?: boolean;
140
+ /** issue #127: time-decay half-life in days; boost = 0.5^(ageDays/halfLife). Omitted = off. */
141
+ recency?: number;
142
+ /** issue #125: query expansion mode. 'prf' is fully offline; 'hyde' needs llmEndpoint (else baseline). */
143
+ expand?: 'prf' | 'hyde';
144
+ /** issue #125: feedback passages for 'prf' (default 3). */
145
+ expandPassages?: number;
146
+ /** issue #125: LLM endpoint used by 'hyde' (OpenAI-compatible chat completions). */
147
+ llmEndpoint?: string;
148
+ /** issue #125: model name for the HyDE chat call. */
149
+ llmModel?: string;
116
150
  ann?: boolean;
117
151
  nprobe?: number;
118
152
  graphBoost?: number;
119
153
  filter?: string | FilterNode;
154
+ dimensions?: number;
120
155
  }
121
156
  export interface SearchResultHit {
122
157
  file: string;