md-semantic-search 1.0.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,104 @@
1
+ /**
2
+ * Pluggable embedding providers (issue #124): Ollama and OpenAI-compatible
3
+ * endpoints beside the built-in local transformers.js path.
4
+ *
5
+ * Design: a provider produces (a) an async embedFn with the SAME signature as
6
+ * core.embed(texts, role, model, cacheDir, offline) and (b) a synthetic
7
+ * ModelAdapter descriptor. The descriptor flows through the EXISTING machinery
8
+ * (adapter fingerprint, dim validation, index metadata) so switching providers
9
+ * invalidates vectors exactly like switching local models — no special cases.
10
+ *
11
+ * Local transformers.js stays the zero-config default and the only fully
12
+ * offline path; providers are opt-in via explicit flags.
13
+ */
14
+ import { loadApiKeyFile } from './serve.js';
15
+ function baseOf(cfg) {
16
+ const raw = cfg.baseUrl ||
17
+ (cfg.embedder === 'ollama' ? process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' : 'https://api.openai.com/v1');
18
+ return raw.replace(/\/$/, '');
19
+ }
20
+ function bearerKey(cfg) {
21
+ if (cfg.embedder !== 'openai')
22
+ return undefined;
23
+ if (cfg.keyFile)
24
+ return loadApiKeyFile(cfg.keyFile);
25
+ return process.env.OPENAI_API_KEY || undefined;
26
+ }
27
+ async function postJson(fetchImpl, url, headers, body) {
28
+ const res = await fetchImpl(url, {
29
+ method: 'POST',
30
+ headers: { 'content-type': 'application/json', ...headers },
31
+ body: JSON.stringify(body),
32
+ });
33
+ if (!res.ok) {
34
+ const text = await res.text().catch(() => '');
35
+ throw new Error(`embedder ${url} -> HTTP ${res.status}${text ? `: ${text.slice(0, 200)}` : ''}`);
36
+ }
37
+ return res.json();
38
+ }
39
+ /** Build an external embedder; dim is probed once on first use and cached. */
40
+ export function resolveExternalEmbedder(cfg) {
41
+ const fetchImpl = cfg.fetchImpl ?? globalThis.fetch;
42
+ if (typeof fetchImpl !== 'function')
43
+ throw new Error('embedder: no fetch implementation available');
44
+ let cachedDim = 0;
45
+ const embedOneBatch = async (texts) => {
46
+ if (cfg.embedder === 'ollama') {
47
+ const data = await postJson(fetchImpl, `${baseOf(cfg)}/api/embed`, {}, { model: cfg.model, input: texts });
48
+ const embs = data?.embeddings;
49
+ if (!Array.isArray(embs) || embs.length !== texts.length) {
50
+ throw new Error('ollama /api/embed returned unexpected shape');
51
+ }
52
+ return embs;
53
+ }
54
+ // OpenAI-compatible
55
+ const headers = {};
56
+ const key = bearerKey(cfg);
57
+ if (key)
58
+ headers.authorization = `Bearer ${key}`;
59
+ const data = await postJson(fetchImpl, `${baseOf(cfg)}/embeddings`, headers, {
60
+ model: cfg.model,
61
+ input: texts,
62
+ });
63
+ const rows = data?.data;
64
+ if (!Array.isArray(rows) || rows.length !== texts.length) {
65
+ throw new Error('openai /embeddings returned unexpected shape');
66
+ }
67
+ return rows
68
+ .slice()
69
+ .sort((a, b) => a.index - b.index)
70
+ .map((r) => r.embedding);
71
+ };
72
+ const embedFn = async (texts, _role) => {
73
+ if (texts.length === 0)
74
+ return [];
75
+ const out = await embedOneBatch(texts);
76
+ if (cachedDim === 0 && out[0]?.length)
77
+ cachedDim = out[0].length;
78
+ return out;
79
+ };
80
+ // Dim probing: one tiny request at resolve time keeps index metadata honest
81
+ // without hardcoding per-model tables.
82
+ const descriptorPromise = (async () => {
83
+ const probe = await embedOneBatch(['dim probe']);
84
+ cachedDim = probe[0]?.length ?? 0;
85
+ if (!cachedDim)
86
+ throw new Error(`embedder ${cfg.embedder}:${cfg.model} returned an empty embedding`);
87
+ return {
88
+ id: `${cfg.embedder}/${cfg.model}`,
89
+ nativeDim: cachedDim,
90
+ dim: cachedDim,
91
+ queryPrefix: '',
92
+ passagePrefix: '',
93
+ pooling: 'mean',
94
+ normalize: false, // never assume; cosine still works on unnormalized vectors
95
+ family: cfg.embedder,
96
+ };
97
+ })();
98
+ return {
99
+ id: cfg.embedder,
100
+ model: cfg.model,
101
+ embedFn,
102
+ descriptor: () => descriptorPromise,
103
+ };
104
+ }
@@ -0,0 +1,14 @@
1
+ export declare const QUERY_CACHE_FILE = "query-cache.json";
2
+ export declare const QUERY_CACHE_MAX_ENTRIES = 512;
3
+ export declare function buildDiskKey(modelId: string, revision: string, dim: number | undefined, query: string): string;
4
+ /**
5
+ * Look up a query vector on disk. Returns null on any miss/corruption —
6
+ * callers fall through to embedding without ceremony.
7
+ */
8
+ export declare function diskQueryGet(cacheDir: string | undefined, key: string, expectedDim: number | undefined): Float32Array | null;
9
+ /** Store a computed query vector (fire-and-forget, debounced flush). */
10
+ export declare function diskQueryPut(cacheDir: string | undefined, key: string, vec: number[] | Float32Array): void;
11
+ /** Test helper: force-write now. */
12
+ export declare function flushQueryCacheNow(cacheDir: string): void;
13
+ /** Test helper: forget in-process state so the next get/put reloads from disk. */
14
+ export declare function resetQueryCacheForTest(cacheDir?: string): void;
@@ -0,0 +1,175 @@
1
+ /**
2
+ * Disk-backed persistent query-embedding cache (issue #114).
3
+ *
4
+ * The in-memory QueryEmbeddingCache only helps long-lived processes; every
5
+ * fresh `mdss search` paid full embedding latency. This module adds an L2
6
+ * layer under it: an LRU (512 entries) persisted to `<cacheDir>/query-cache.json`
7
+ * keyed by model@revision|dim|sha256(query) — the same identity the memory
8
+ * cache uses, hashed for compactness.
9
+ *
10
+ * Integrity mirrors vectors.bin: atomic tmp+rename write plus a `.sha256`
11
+ * sidecar. A missing/corrupt pair is silently ignored and rebuilt — a broken
12
+ * cache must never fail a search.
13
+ */
14
+ import crypto from 'node:crypto';
15
+ import fs from 'node:fs';
16
+ import path from 'node:path';
17
+ export const QUERY_CACHE_FILE = 'query-cache.json';
18
+ export const QUERY_CACHE_MAX_ENTRIES = 512;
19
+ export function buildDiskKey(modelId, revision, dim, query) {
20
+ const norm = query.trim().toLowerCase();
21
+ return `${modelId}@${revision || 'main'}|${dim ?? 0}|${crypto.createHash('sha256').update(norm).digest('hex')}`;
22
+ }
23
+ /** Per-directory in-process state: loaded entries + dirty flag. */
24
+ const states = new Map();
25
+ function stateFor(cacheDir) {
26
+ let st = states.get(cacheDir);
27
+ if (!st) {
28
+ st = { entries: loadFromDisk(cacheDir), dirty: false };
29
+ states.set(cacheDir, st);
30
+ }
31
+ return st;
32
+ }
33
+ function filePaths(cacheDir) {
34
+ const file = path.join(cacheDir, QUERY_CACHE_FILE);
35
+ return { file, sha: `${file}.sha256`, tmp: `${file}.tmp-${process.pid}` };
36
+ }
37
+ function loadFromDisk(cacheDir) {
38
+ const out = new Map();
39
+ const { file, sha } = filePaths(cacheDir);
40
+ let raw;
41
+ try {
42
+ raw = fs.readFileSync(file);
43
+ }
44
+ catch {
45
+ return out;
46
+ }
47
+ try {
48
+ if (fs.existsSync(sha)) {
49
+ const expected = fs.readFileSync(sha, 'utf8').trim().split(/\s+/)[0];
50
+ const actual = crypto.createHash('sha256').update(raw).digest('hex');
51
+ if (!expected || expected !== actual)
52
+ return out; // corrupt → silent rebuild
53
+ }
54
+ const parsed = JSON.parse(raw.toString('utf8'));
55
+ if (parsed.version !== 1 || typeof parsed.entries !== 'object')
56
+ return out;
57
+ for (const [key, e] of Object.entries(parsed.entries)) {
58
+ if (typeof key === 'string' &&
59
+ typeof e?.vec === 'string' &&
60
+ Number.isInteger(e?.dim) &&
61
+ e.dim > 0 &&
62
+ Number.isFinite(e?.ts)) {
63
+ out.set(key, { vec: e.vec, dim: e.dim, ts: e.ts });
64
+ }
65
+ }
66
+ }
67
+ catch {
68
+ return new Map(); // any corruption → treat as empty; next flush overwrites
69
+ }
70
+ return out;
71
+ }
72
+ function flushSync(cacheDir) {
73
+ const st = states.get(cacheDir);
74
+ if (!st || !st.dirty)
75
+ return;
76
+ // LRU trim before writing
77
+ const sorted = [...st.entries.entries()].sort((a, b) => b[1].ts - a[1].ts);
78
+ st.entries = new Map(sorted.slice(0, QUERY_CACHE_MAX_ENTRIES));
79
+ const payload = {
80
+ version: 1,
81
+ entries: Object.fromEntries(st.entries),
82
+ };
83
+ const body = Buffer.from(JSON.stringify(payload));
84
+ const digest = crypto.createHash('sha256').update(body).digest('hex');
85
+ const { file, sha, tmp } = filePaths(cacheDir);
86
+ try {
87
+ fs.mkdirSync(cacheDir, { recursive: true });
88
+ fs.writeFileSync(tmp, body);
89
+ fs.renameSync(tmp, file);
90
+ fs.writeFileSync(`${sha}.tmp`, `${digest} ${QUERY_CACHE_FILE}\n`);
91
+ fs.renameSync(`${sha}.tmp`, sha);
92
+ st.dirty = false;
93
+ }
94
+ catch {
95
+ /* unwritable cacheDir is non-fatal by design */
96
+ }
97
+ }
98
+ function scheduleFlush(cacheDir) {
99
+ const st = stateFor(cacheDir);
100
+ if (st.timer)
101
+ return;
102
+ st.timer = setTimeout(() => {
103
+ st.timer = undefined;
104
+ flushSync(cacheDir);
105
+ }, 400);
106
+ st.timer.unref?.();
107
+ }
108
+ // best-effort flush on process exit so one-shot CLI runs persist their queries
109
+ let exitHookInstalled = false;
110
+ function ensureExitHook() {
111
+ if (exitHookInstalled)
112
+ return;
113
+ exitHookInstalled = true;
114
+ process.on('exit', () => {
115
+ for (const dir of states.keys())
116
+ flushSync(dir);
117
+ });
118
+ }
119
+ /**
120
+ * Look up a query vector on disk. Returns null on any miss/corruption —
121
+ * callers fall through to embedding without ceremony.
122
+ */
123
+ export function diskQueryGet(cacheDir, key, expectedDim) {
124
+ if (!cacheDir)
125
+ return null;
126
+ const st = stateFor(cacheDir);
127
+ const e = st.entries.get(key);
128
+ if (!e)
129
+ return null;
130
+ try {
131
+ const buf = Buffer.from(e.vec, 'base64');
132
+ if (buf.byteLength !== e.dim * 4)
133
+ return null;
134
+ const arr = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
135
+ if (expectedDim !== undefined && arr.length !== expectedDim)
136
+ return null;
137
+ for (const x of arr)
138
+ if (!Number.isFinite(x))
139
+ return null;
140
+ e.ts = Date.now(); // LRU touch
141
+ st.dirty = true;
142
+ scheduleFlush(cacheDir);
143
+ return arr;
144
+ }
145
+ catch {
146
+ st.entries.delete(key);
147
+ return null;
148
+ }
149
+ }
150
+ /** Store a computed query vector (fire-and-forget, debounced flush). */
151
+ export function diskQueryPut(cacheDir, key, vec) {
152
+ if (!cacheDir)
153
+ return;
154
+ const f32 = vec instanceof Float32Array ? vec : Float32Array.from(vec);
155
+ const st = stateFor(cacheDir);
156
+ st.entries.set(key, {
157
+ vec: Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength).toString('base64'),
158
+ dim: f32.length,
159
+ ts: Date.now(),
160
+ });
161
+ st.dirty = true;
162
+ ensureExitHook();
163
+ scheduleFlush(cacheDir);
164
+ }
165
+ /** Test helper: force-write now. */
166
+ export function flushQueryCacheNow(cacheDir) {
167
+ flushSync(cacheDir);
168
+ }
169
+ /** Test helper: forget in-process state so the next get/put reloads from disk. */
170
+ export function resetQueryCacheForTest(cacheDir) {
171
+ if (cacheDir)
172
+ states.delete(cacheDir);
173
+ else
174
+ states.clear();
175
+ }
@@ -0,0 +1,36 @@
1
+ export interface RepairAction {
2
+ action: 'remove-stale-lock' | 'remove-broken-vectors-bin' | 'remove-corrupt-ivf' | 'rebuild-index' | 'rewrite-hashes';
3
+ detail: string;
4
+ }
5
+ export interface RepairPlan {
6
+ actions: RepairAction[];
7
+ skipped: RepairAction[];
8
+ }
9
+ export interface ApplyResult {
10
+ performed: RepairAction[];
11
+ failed: Array<RepairAction & {
12
+ error: string;
13
+ }>;
14
+ }
15
+ interface ProbeOptions {
16
+ db?: string | null;
17
+ log?: (msg: string) => void;
18
+ }
19
+ /** Recompute the {relativePath: md5} map exactly as the indexer stores it. */
20
+ export declare function recomputeHashes(db: string): Record<string, string>;
21
+ /**
22
+ * Probe the index directory and decide what CAN be safely repaired.
23
+ * Never mutates anything — safe to call for --dry-run.
24
+ */
25
+ export declare function planRepairs(indexDir: string, opts?: ProbeOptions): RepairPlan;
26
+ /** Execute planned actions in order. dryRun=true performs nothing. */
27
+ export declare function applyRepairs(indexDir: string, plan: RepairPlan, opts?: {
28
+ dryRun?: boolean;
29
+ db?: string | null;
30
+ cacheDir?: string;
31
+ modelName?: string;
32
+ embedFn?: any;
33
+ offline?: boolean;
34
+ log?: (msg: string) => void;
35
+ }): Promise<ApplyResult>;
36
+ export {};
package/dist/repair.js ADDED
@@ -0,0 +1,208 @@
1
+ /**
2
+ * Safe auto-repairs for `mdss check --fix` (issue #117).
3
+ *
4
+ * Plan/apply split: planRepairs() probes the index directory WITHOUT mutating
5
+ * anything (so --dry-run can print the exact plan), applyRepairs() executes
6
+ * actions in order and reports per-action outcomes. Source Markdown is NEVER
7
+ * touched; the only reads from --db are for recomputing `.hashes.json` or a
8
+ * full index rebuild.
9
+ */
10
+ import crypto from 'node:crypto';
11
+ import fs from 'node:fs';
12
+ import path from 'node:path';
13
+ import { LOCK_FILENAME, pidAlive, walkMarkdown } from './core.js';
14
+ import { DEFAULT_MODEL } from './models.js';
15
+ const LOCK_STALE_MS = 10 * 60 * 1000;
16
+ function readJsonIfParsable(filePath) {
17
+ let raw;
18
+ try {
19
+ raw = fs.readFileSync(filePath, 'utf8');
20
+ }
21
+ catch {
22
+ return { ok: 'missing' };
23
+ }
24
+ try {
25
+ return { ok: true, value: JSON.parse(raw) };
26
+ }
27
+ catch {
28
+ return { ok: false };
29
+ }
30
+ }
31
+ /** Recompute the {relativePath: md5} map exactly as the indexer stores it. */
32
+ export function recomputeHashes(db) {
33
+ const hashes = {};
34
+ for (const f of walkMarkdown(db)) {
35
+ const rel = path.relative(db, f).split(path.sep).join('/');
36
+ const md5 = crypto.createHash('md5').update(fs.readFileSync(f)).digest('hex');
37
+ hashes[rel] = md5;
38
+ }
39
+ return hashes;
40
+ }
41
+ /**
42
+ * Probe the index directory and decide what CAN be safely repaired.
43
+ * Never mutates anything — safe to call for --dry-run.
44
+ */
45
+ export function planRepairs(indexDir, opts = {}) {
46
+ const plan = { actions: [], skipped: [] };
47
+ const db = opts.db ?? null;
48
+ // --- 1. stale / abandoned lock ---
49
+ const lockPath = path.join(indexDir, LOCK_FILENAME);
50
+ if (fs.existsSync(lockPath)) {
51
+ let info = null;
52
+ try {
53
+ info = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
54
+ }
55
+ catch {
56
+ info = null;
57
+ }
58
+ const pid = Number.isInteger(info?.pid) ? info.pid : null;
59
+ let statAgeMs = Infinity;
60
+ try {
61
+ statAgeMs = Date.now() - fs.statSync(lockPath).mtimeMs;
62
+ }
63
+ catch {
64
+ /* vanished between exists and stat */
65
+ }
66
+ if (!info) {
67
+ plan.actions.push({ action: 'remove-stale-lock', detail: `${LOCK_FILENAME} is unreadable/abandoned` });
68
+ }
69
+ else if (pid !== null && !pidAlive(pid)) {
70
+ plan.actions.push({ action: 'remove-stale-lock', detail: `${LOCK_FILENAME} held by dead pid ${pid}` });
71
+ }
72
+ else if (statAgeMs > LOCK_STALE_MS) {
73
+ plan.actions.push({
74
+ action: 'remove-stale-lock',
75
+ detail: `${LOCK_FILENAME} is older than ${Math.round(LOCK_STALE_MS / 60000)} min`,
76
+ });
77
+ }
78
+ else {
79
+ plan.skipped.push({
80
+ action: 'remove-stale-lock',
81
+ detail: `live writer holds the lock${pid ? ` (pid ${pid})` : ''}`,
82
+ });
83
+ }
84
+ }
85
+ // --- 2. vectors.bin integrity (SHA-256 sidecar) ---
86
+ const binPath = path.join(indexDir, 'vectors.bin');
87
+ const shaPath = `${binPath}.sha256`;
88
+ if (fs.existsSync(binPath)) {
89
+ let broken = null;
90
+ try {
91
+ const actual = crypto.createHash('sha256').update(fs.readFileSync(binPath)).digest('hex');
92
+ if (!fs.existsSync(shaPath)) {
93
+ broken = 'missing vectors.bin.sha256 sidecar';
94
+ }
95
+ else {
96
+ const expected = fs.readFileSync(shaPath, 'utf8').trim().split(/\s+/)[0];
97
+ if (!expected || expected !== actual)
98
+ broken = 'SHA-256 mismatch';
99
+ }
100
+ }
101
+ catch (e) {
102
+ broken = e.message;
103
+ }
104
+ if (broken) {
105
+ plan.actions.push({
106
+ action: 'remove-broken-vectors-bin',
107
+ detail: `vectors.bin ignored (${broken}) — search falls back to vectors.json`,
108
+ });
109
+ }
110
+ }
111
+ // --- 3. corrupt ivf.json (optional ANN sidecar; loader silently ignores it) ---
112
+ const ivfPath = path.join(indexDir, 'ivf.json');
113
+ const ivfRaw = readJsonIfParsable(ivfPath);
114
+ if (ivfRaw.ok === false) {
115
+ plan.actions.push({
116
+ action: 'remove-corrupt-ivf',
117
+ detail: 'ivf.json is not valid JSON — removing it re-enables brute-force ANN fallback',
118
+ });
119
+ }
120
+ // --- 4. corrupt vectors.json ---
121
+ const liveWriter = plan.skipped.some((s) => s.action === 'remove-stale-lock' && /live writer/.test(s.detail));
122
+ const vectorsRaw = readJsonIfParsable(path.join(indexDir, 'vectors.json'));
123
+ if (vectorsRaw.ok === false) {
124
+ if (db && !liveWriter) {
125
+ plan.actions.push({ action: 'rebuild-index', detail: `vectors.json is not valid JSON — rebuild from ${db}` });
126
+ }
127
+ else if (liveWriter) {
128
+ plan.skipped.push({
129
+ action: 'rebuild-index',
130
+ detail: 'vectors.json is not valid JSON, but a live writer holds the lock — retry after it finishes',
131
+ });
132
+ }
133
+ else {
134
+ plan.skipped.push({
135
+ action: 'rebuild-index',
136
+ detail: 'vectors.json is not valid JSON — pass --db so --fix can rebuild it',
137
+ });
138
+ }
139
+ }
140
+ // --- 5. damaged .hashes.json ---
141
+ const hashesRaw = readJsonIfParsable(path.join(indexDir, '.hashes.json'));
142
+ if (hashesRaw.ok === false) {
143
+ if (db) {
144
+ plan.actions.push({ action: 'rewrite-hashes', detail: '.hashes.json is not valid JSON — recompute from source md5' });
145
+ }
146
+ else {
147
+ plan.skipped.push({
148
+ action: 'rewrite-hashes',
149
+ detail: '.hashes.json is not valid JSON — pass --db so --fix can recompute it',
150
+ });
151
+ }
152
+ }
153
+ return plan;
154
+ }
155
+ /** Execute planned actions in order. dryRun=true performs nothing. */
156
+ export function applyRepairs(indexDir, plan, opts = {}) {
157
+ const performed = [];
158
+ const failed = [];
159
+ const log = opts.log ?? (() => { });
160
+ const run = async () => {
161
+ for (const action of plan.actions) {
162
+ if (opts.dryRun) {
163
+ performed.push(action);
164
+ continue;
165
+ }
166
+ try {
167
+ switch (action.action) {
168
+ case 'remove-stale-lock':
169
+ fs.unlinkSync(path.join(indexDir, LOCK_FILENAME));
170
+ break;
171
+ case 'remove-broken-vectors-bin':
172
+ fs.rmSync(path.join(indexDir, 'vectors.bin'), { force: true });
173
+ fs.rmSync(`${path.join(indexDir, 'vectors.bin')}.sha256`, { force: true });
174
+ break;
175
+ case 'remove-corrupt-ivf':
176
+ fs.rmSync(path.join(indexDir, 'ivf.json'), { force: true });
177
+ break;
178
+ case 'rewrite-hashes': {
179
+ const hashes = recomputeHashes(opts.db);
180
+ fs.writeFileSync(path.join(indexDir, '.hashes.json'), JSON.stringify(hashes, null, 2));
181
+ break;
182
+ }
183
+ case 'rebuild-index': {
184
+ const { buildIndex } = await import('./indexer.js');
185
+ const rebuildOpts = {
186
+ db: opts.db,
187
+ indexDir,
188
+ cacheDir: opts.cacheDir,
189
+ modelName: opts.modelName || DEFAULT_MODEL,
190
+ log,
191
+ _lockHeld: true, // we may have JUST removed a stale lock; never double-lock inside fix
192
+ };
193
+ if (opts.embedFn)
194
+ rebuildOpts.embedFn = opts.embedFn; // test DI — keeps repairs model-free
195
+ await buildIndex(rebuildOpts);
196
+ break;
197
+ }
198
+ }
199
+ performed.push(action);
200
+ }
201
+ catch (e) {
202
+ failed.push({ ...action, error: e.message });
203
+ }
204
+ }
205
+ return { performed, failed };
206
+ };
207
+ return run();
208
+ }
package/dist/rerank.js CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Cross-encoder re-ranking (issue #15).
3
3
  */
4
- import { resolveSessionOptions, retryWithBackoff } from './core.js';
4
+ import { resolveSessionOptions, retryWithBackoff, emitDownloadEvent } from './core.js';
5
5
  const _rerankers = new Map();
6
6
  export const RERANK_MODEL = 'Xenova/bge-reranker-base';
7
7
  export async function getReranker(cacheDir, offline = false, retryOpts = {}) {
@@ -15,9 +15,10 @@ export async function getReranker(cacheDir, offline = false, retryOpts = {}) {
15
15
  if (cacheDir)
16
16
  env.cacheDir = cacheDir;
17
17
  env.allowRemoteModels = !offline;
18
- const tokenizer = await AutoTokenizer.from_pretrained(RERANK_MODEL);
18
+ const tokenizer = await AutoTokenizer.from_pretrained(RERANK_MODEL, { progress_callback: emitDownloadEvent });
19
19
  const loadOpts = {
20
20
  dtype: 'q8',
21
+ progress_callback: emitDownloadEvent,
21
22
  ...(sessionOptions ? { session_options: sessionOptions } : {}),
22
23
  };
23
24
  const model = await AutoModelForSequenceClassification.from_pretrained(RERANK_MODEL, loadOpts);
package/dist/search.d.ts CHANGED
@@ -81,6 +81,28 @@ export declare function getRuntimeGraph(runtime: RuntimeIndexState): {
81
81
  graph: RelationshipGraph;
82
82
  pageRank: Map<string, number>;
83
83
  };
84
+ /**
85
+ * Issue #127 pure helpers — exported for deterministic unit tests.
86
+ */
87
+ /** Frontmatter `created` → `updated` → file mtime; null when nothing parses. */
88
+ export declare function resolvePassageAgeMs(meta: {
89
+ created?: unknown;
90
+ updated?: unknown;
91
+ } | undefined, absFile?: string): number | null;
92
+ /** Multiplicative recency boost: 0.5^(ageDays / halfLifeDays); 1 when off/unknown/future. */
93
+ export declare function recencyBoost(ageMs: number | null, halfLifeDays: number, now?: number): number;
94
+ /**
95
+ * HyDE passage generation via an OpenAI-compatible chat endpoint (issue #125).
96
+ * Returns null on ANY failure — HyDE must degrade to PRF/baseline silently.
97
+ * Injectable fetch for tests.
98
+ */
99
+ export declare function hydePassage(endpoint: string, query: string, model?: string, fetchImpl?: typeof fetch): Promise<string | null>;
100
+ /**
101
+ * Salient-term extraction for PRF (issue #125): tokens that appear in the
102
+ * top-m feedback passages, are not query terms or stopwords, and are ranked
103
+ * by (tf-in-feedback × distinct-passage coverage). Pure + deterministic.
104
+ */
105
+ export declare function extractExpansionTerms(topTexts: string[], queryTerms: string[], maxTerms: number): string[];
84
106
  export declare function loadIndex(indexDir: string): {
85
107
  index: IndexFile;
86
108
  model: ModelDescriptor;
@@ -113,6 +135,18 @@ export interface SearchOptions {
113
135
  maxPerFile?: number;
114
136
  maxPerDoc?: number;
115
137
  useQueryCache?: boolean;
138
+ /** issue #114: persist query embeddings to <cacheDir>/query-cache.json (default on). */
139
+ queryDiskCache?: boolean;
140
+ /** issue #127: time-decay half-life in days; boost = 0.5^(ageDays/halfLife). Omitted = off. */
141
+ recency?: number;
142
+ /** issue #125: query expansion mode. 'prf' is fully offline; 'hyde' needs llmEndpoint (else baseline). */
143
+ expand?: 'prf' | 'hyde';
144
+ /** issue #125: feedback passages for 'prf' (default 3). */
145
+ expandPassages?: number;
146
+ /** issue #125: LLM endpoint used by 'hyde' (OpenAI-compatible chat completions). */
147
+ llmEndpoint?: string;
148
+ /** issue #125: model name for the HyDE chat call. */
149
+ llmModel?: string;
116
150
  ann?: boolean;
117
151
  nprobe?: number;
118
152
  graphBoost?: number;