md-semantic-search 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -6
- package/bin/cli.mjs +403 -22
- package/dist/completions.d.ts +25 -0
- package/dist/completions.js +233 -0
- package/dist/core.d.ts +5 -0
- package/dist/core.js +17 -0
- package/dist/download-progress.d.ts +52 -0
- package/dist/download-progress.js +112 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/mcp.js +6 -1
- package/dist/open.d.ts +43 -0
- package/dist/open.js +84 -0
- package/dist/providers.d.ts +21 -0
- package/dist/providers.js +104 -0
- package/dist/query-cache-disk.d.ts +14 -0
- package/dist/query-cache-disk.js +175 -0
- package/dist/repair.d.ts +36 -0
- package/dist/repair.js +208 -0
- package/dist/rerank.js +3 -2
- package/dist/search.d.ts +34 -0
- package/dist/search.js +172 -5
- package/dist/serve.d.ts +126 -0
- package/dist/serve.js +579 -45
- package/dist/watcher.d.ts +16 -0
- package/dist/watcher.js +41 -0
- package/dist/webui.d.ts +13 -0
- package/dist/webui.js +231 -0
- package/package.json +1 -1
- package/src/completions.ts +250 -0
- package/src/core.ts +26 -0
- package/src/download-progress.ts +146 -0
- package/src/index.ts +4 -0
- package/src/mcp.ts +10 -3
- package/src/open.ts +119 -0
- package/src/providers.ts +134 -0
- package/src/query-cache-disk.ts +191 -0
- package/src/repair.ts +245 -0
- package/src/rerank.ts +3 -2
- package/src/search.ts +192 -6
- package/src/serve.ts +635 -42
- package/src/watcher.ts +46 -0
- package/src/webui.ts +234 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Aggregated model-download progress (issue #108).
|
|
3
|
+
*
|
|
4
|
+
* transformers.js emits per-file events (`initiate`/`download`/`progress`/`done`)
|
|
5
|
+
* while a model directory streams in. A model is many files, so a per-file bar
|
|
6
|
+
* flickers uselessly — this aggregator folds them into ONE view: percent across
|
|
7
|
+
* every tracked file, smoothed MB/s, and an ETA over the bytes still missing.
|
|
8
|
+
*
|
|
9
|
+
* The "sawPartial" gate implements the warm-cache acceptance criterion: cached
|
|
10
|
+
* loads surface only instant complete-file events, so unless a real partial
|
|
11
|
+
* transfer was observed the aggregator stays silent.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export interface DownloadProgressEvent {
|
|
15
|
+
status?: string;
|
|
16
|
+
name?: string;
|
|
17
|
+
file?: string;
|
|
18
|
+
loaded?: number;
|
|
19
|
+
total?: number;
|
|
20
|
+
progress?: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface DownloadSnapshot {
|
|
24
|
+
/** True once a partial transfer was observed — gates ALL output. */
|
|
25
|
+
reportable: boolean;
|
|
26
|
+
/** True when every file with a known total has fully arrived. */
|
|
27
|
+
complete: boolean;
|
|
28
|
+
doneFiles: number;
|
|
29
|
+
activeFiles: number;
|
|
30
|
+
loadedBytes: number;
|
|
31
|
+
totalBytes: number;
|
|
32
|
+
/** 0..100 across files with known totals; null while no total is known yet. */
|
|
33
|
+
percent: number | null;
|
|
34
|
+
mbps: number | null;
|
|
35
|
+
etaSec: number | null;
|
|
36
|
+
currentFile: string | null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface FileProgress {
|
|
40
|
+
loaded: number;
|
|
41
|
+
total: number; // 0 = unknown
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const SPEED_SMOOTHING = 0.3; // EMA weight of the newest instantaneous sample
|
|
45
|
+
|
|
46
|
+
export class DownloadProgressAggregator {
|
|
47
|
+
private files = new Map<string, FileProgress>();
|
|
48
|
+
private sawPartial = false;
|
|
49
|
+
private lastSampleAt = 0;
|
|
50
|
+
private lastSampleBytes = 0;
|
|
51
|
+
private mbpsEma: number | null = null;
|
|
52
|
+
private lastRenderAt = 0;
|
|
53
|
+
|
|
54
|
+
update(e: DownloadProgressEvent, nowMs: number = Date.now()): void {
|
|
55
|
+
if (!e || typeof e.file !== 'string' || e.file.length === 0) return;
|
|
56
|
+
let f = this.files.get(e.file);
|
|
57
|
+
if (!f) {
|
|
58
|
+
f = { loaded: 0, total: 0 };
|
|
59
|
+
this.files.set(e.file, f);
|
|
60
|
+
}
|
|
61
|
+
if (typeof e.total === 'number' && Number.isFinite(e.total) && e.total > f.total) {
|
|
62
|
+
f.total = e.total;
|
|
63
|
+
}
|
|
64
|
+
if (typeof e.loaded === 'number' && Number.isFinite(e.loaded) && e.loaded >= 0) {
|
|
65
|
+
// clamp against the known total so a stray event can't overshoot to >100%
|
|
66
|
+
f.loaded = f.total > 0 ? Math.min(e.loaded, f.total) : e.loaded;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const midTransfer =
|
|
70
|
+
(e.status === 'download' || e.status === 'progress') && f.total > 0 && f.loaded < f.total;
|
|
71
|
+
if (midTransfer) this.sawPartial = true;
|
|
72
|
+
|
|
73
|
+
// speed sampling on cumulative byte growth
|
|
74
|
+
const sum = this.sumLoaded();
|
|
75
|
+
if (this.lastSampleAt === 0) {
|
|
76
|
+
this.lastSampleAt = nowMs;
|
|
77
|
+
this.lastSampleBytes = sum;
|
|
78
|
+
} else if (sum > this.lastSampleBytes) {
|
|
79
|
+
const dtSec = (nowMs - this.lastSampleAt) / 1000;
|
|
80
|
+
if (dtSec > 0.05) {
|
|
81
|
+
const inst = (sum - this.lastSampleBytes) / dtSec / (1024 * 1024);
|
|
82
|
+
this.mbpsEma = this.mbpsEma === null ? inst : this.mbpsEma * (1 - SPEED_SMOOTHING) + inst * SPEED_SMOOTHING;
|
|
83
|
+
this.lastSampleAt = nowMs;
|
|
84
|
+
this.lastSampleBytes = sum;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
private sumLoaded(): number {
|
|
90
|
+
let sum = 0;
|
|
91
|
+
for (const f of this.files.values()) sum += f.loaded;
|
|
92
|
+
return sum;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
snapshot(): DownloadSnapshot {
|
|
96
|
+
let loadedBytes = 0;
|
|
97
|
+
let totalBytes = 0;
|
|
98
|
+
let doneFiles = 0;
|
|
99
|
+
let activeFiles = 0;
|
|
100
|
+
for (const f of this.files.values()) {
|
|
101
|
+
loadedBytes += f.loaded;
|
|
102
|
+
if (f.total > 0) {
|
|
103
|
+
totalBytes += f.total;
|
|
104
|
+
if (f.loaded >= f.total) doneFiles += 1;
|
|
105
|
+
else activeFiles += 1;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
const percent = totalBytes > 0 ? Math.min(100, (loadedBytes / totalBytes) * 100) : null;
|
|
109
|
+
const remaining = totalBytes - loadedBytes;
|
|
110
|
+
const etaSec =
|
|
111
|
+
this.mbpsEma !== null && this.mbpsEma > 0 && remaining > 0 ? remaining / (1024 * 1024) / this.mbpsEma : null;
|
|
112
|
+
return {
|
|
113
|
+
reportable: this.sawPartial,
|
|
114
|
+
complete: totalBytes > 0 && activeFiles === 0,
|
|
115
|
+
doneFiles,
|
|
116
|
+
activeFiles,
|
|
117
|
+
loadedBytes,
|
|
118
|
+
totalBytes,
|
|
119
|
+
percent,
|
|
120
|
+
mbps: this.sawPartial ? this.mbpsEma : null,
|
|
121
|
+
etaSec: this.sawPartial ? etaSec : null,
|
|
122
|
+
currentFile: [...this.files.entries()].find(([, f]) => f.total === 0 || f.loaded < f.total)?.[0] ?? null,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Render throttle for live views: pass through at most one line per
|
|
128
|
+
* interval, plus ALWAYS the transition into `complete`.
|
|
129
|
+
*/
|
|
130
|
+
shouldRender(nowMs: number, minIntervalMs = 400): boolean {
|
|
131
|
+
if (!this.sawPartial) return false;
|
|
132
|
+
const justCompleted = this.snapshot().complete && !this.renderedComplete;
|
|
133
|
+
if (justCompleted) {
|
|
134
|
+
this.renderedComplete = true;
|
|
135
|
+
this.lastRenderAt = nowMs;
|
|
136
|
+
return true;
|
|
137
|
+
}
|
|
138
|
+
if (nowMs - this.lastRenderAt >= minIntervalMs) {
|
|
139
|
+
this.lastRenderAt = nowMs;
|
|
140
|
+
return true;
|
|
141
|
+
}
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
private renderedComplete = false;
|
|
146
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -89,5 +89,9 @@ export {
|
|
|
89
89
|
} from './summarize.js';
|
|
90
90
|
|
|
91
91
|
export { runTui, TuiOptions } from './tui.js';
|
|
92
|
+
export {
|
|
93
|
+
generateCompletion, FLAGS, COMMANDS, COMPLETION_SHELLS,
|
|
94
|
+
} from './completions.js';
|
|
95
|
+
export type { FlagSpec, CompletionShell } from './completions.js';
|
|
92
96
|
|
|
93
97
|
|
package/src/mcp.ts
CHANGED
|
@@ -12,6 +12,14 @@ import { searchFederated } from './federation.js';
|
|
|
12
12
|
import { askQuestion } from './rag.js';
|
|
13
13
|
import { globToRegExp } from './core.js';
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* Package version reported via MCP initialize → serverInfo.
|
|
17
|
+
* Single source of truth: package.json (resolved relative to dist/mcp.js).
|
|
18
|
+
*/
|
|
19
|
+
const MCP_VERSION: string = JSON.parse(
|
|
20
|
+
fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8'),
|
|
21
|
+
).version;
|
|
22
|
+
|
|
15
23
|
export const MCP_TOOLS = [
|
|
16
24
|
{
|
|
17
25
|
name: 'search_markdown',
|
|
@@ -74,8 +82,7 @@ export const MCP_TOOLS = [
|
|
|
74
82
|
},
|
|
75
83
|
];
|
|
76
84
|
|
|
77
|
-
export async function handleMcpRequest(req: any, state: { loaded: any; cacheDir: string; offline: boolean; embedFn?: any }): Promise<any | null> {
|
|
78
|
-
if (!req || typeof req !== 'object') return null;
|
|
85
|
+
export async function handleMcpRequest(req: any, state: { loaded: any; cacheDir: string; offline: boolean; embedFn?: any }): Promise<any | null> { if (!req || typeof req !== 'object') return null;
|
|
79
86
|
const { id, method, params } = req;
|
|
80
87
|
|
|
81
88
|
if (id === undefined || id === null) {
|
|
@@ -93,7 +100,7 @@ export async function handleMcpRequest(req: any, state: { loaded: any; cacheDir:
|
|
|
93
100
|
},
|
|
94
101
|
serverInfo: {
|
|
95
102
|
name: 'md-semantic-search',
|
|
96
|
-
version:
|
|
103
|
+
version: MCP_VERSION,
|
|
97
104
|
},
|
|
98
105
|
},
|
|
99
106
|
};
|
package/src/open.ts
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Editor jump for search hits (issue #110): open a hit's file at its exact
|
|
3
|
+
* `startLine` in $MDSS_EDITOR / $VISUAL / $EDITOR, with smart argument styles
|
|
4
|
+
* for known editor families and a platform GUI opener as the last resort.
|
|
5
|
+
*
|
|
6
|
+
* Resolution is PURE (unit-testable); launching goes through an injected
|
|
7
|
+
* runner so tests never spawn real processes.
|
|
8
|
+
*/
|
|
9
|
+
import { spawn } from 'node:child_process';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
|
|
12
|
+
export interface OpenTarget {
|
|
13
|
+
file: string;
|
|
14
|
+
line?: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface OpenCommand {
|
|
18
|
+
command: string;
|
|
19
|
+
args: string[];
|
|
20
|
+
/** Human-readable source of the decision, e.g. "$EDITOR=vim" or "GUI opener". */
|
|
21
|
+
via: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface ResolveOptions {
|
|
25
|
+
/** $MDSS_EDITOR / $VISUAL / $EDITOR value, if any. */
|
|
26
|
+
editor?: string | null;
|
|
27
|
+
platform?: NodeJS.Platform;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface RunOptions extends ResolveOptions {
|
|
31
|
+
runner?: OpenRunner;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export type OpenRunner = (
|
|
35
|
+
command: string,
|
|
36
|
+
args: string[],
|
|
37
|
+
opts: { detached: boolean; stdio: 'ignore' }
|
|
38
|
+
) => unknown;
|
|
39
|
+
|
|
40
|
+
const VSCODE_FAMILY = /^(code|code-insiders|codium|cursor|windsurf)$/i;
|
|
41
|
+
/** Terminal editors taking a "+<line>" positional before the file. */
|
|
42
|
+
const PLUS_LINE_EDITORS = /^(vim|vi|nvim|nano|ne|emacs|emacsclient|micro|hx)$/i;
|
|
43
|
+
|
|
44
|
+
function baseName(cmd: string): string {
|
|
45
|
+
// handle BOTH separators so a Windows full path resolves its family even
|
|
46
|
+
// when mdss runs on POSIX CI (issue #110)
|
|
47
|
+
const last = cmd.split(/[\\/]/).pop() ?? cmd;
|
|
48
|
+
return last.replace(/\.(exe|cmd|bat)$/i, '');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function splitEditorCommand(editor: string): string[] {
|
|
52
|
+
// A value containing a path separator is ONE binary path (Windows installs
|
|
53
|
+
// live under "C:\Program Files\..."); only bare names may carry extra flags.
|
|
54
|
+
if (/[/\\]/.test(editor)) return [editor];
|
|
55
|
+
return editor.trim().split(/\s+/);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Decide HOW to open `target`.
|
|
60
|
+
* - editor given: first token is the binary, remaining tokens are forwarded
|
|
61
|
+
* verbatim. VS Code family gets `--goto file:line`, plus-line editors get
|
|
62
|
+
* `+<line> file`, anything else just gets the file.
|
|
63
|
+
* - No editor configured → platform GUI opener (win32 `cmd /c start ""`,
|
|
64
|
+
* darwin `open`, else `xdg-open`); a line cannot be passed there.
|
|
65
|
+
* Returns null only on an unsupported platform without an editor.
|
|
66
|
+
*/
|
|
67
|
+
export function resolveOpenCommand(target: OpenTarget, opts: ResolveOptions = {}): OpenCommand | null {
|
|
68
|
+
const { editor, platform = process.platform } = opts;
|
|
69
|
+
const line = Number.isInteger(target.line) && (target.line as number) > 0 ? target.line : undefined;
|
|
70
|
+
|
|
71
|
+
if (editor && editor.trim()) {
|
|
72
|
+
const parts = splitEditorCommand(editor.trim());
|
|
73
|
+
const command = parts[0];
|
|
74
|
+
const extra = parts.slice(1);
|
|
75
|
+
const name = baseName(command);
|
|
76
|
+
if (VSCODE_FAMILY.test(name)) {
|
|
77
|
+
return {
|
|
78
|
+
command,
|
|
79
|
+
args: [...extra, '--goto', `${target.file}${line ? `:${line}` : ''}`],
|
|
80
|
+
via: `$EDITOR=${editor}`,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
if (PLUS_LINE_EDITORS.test(name)) {
|
|
84
|
+
return { command, args: [...extra, ...(line ? [`+${line}`] : []), target.file], via: `$EDITOR=${editor}` };
|
|
85
|
+
}
|
|
86
|
+
return { command, args: [...extra, target.file], via: `$EDITOR=${editor}` };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (platform === 'win32') {
|
|
90
|
+
// empty title arg prevents `start` from mangling quoted paths
|
|
91
|
+
return { command: 'cmd', args: ['/c', 'start', '', target.file], via: 'GUI opener' };
|
|
92
|
+
}
|
|
93
|
+
if (platform === 'darwin') {
|
|
94
|
+
return { command: 'open', args: [target.file], via: 'GUI opener' };
|
|
95
|
+
}
|
|
96
|
+
if (platform === 'linux' || platform === 'freebsd' || platform === 'openbsd') {
|
|
97
|
+
return { command: 'xdg-open', args: [target.file], via: 'GUI opener' };
|
|
98
|
+
}
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Launch via the default detached spawn; inject a fake in tests. */
|
|
103
|
+
export function defaultOpenRunner(command: string, args: string[], opts: { detached: boolean; stdio: 'ignore' }): void {
|
|
104
|
+
const child = spawn(command, args, { detached: true, stdio: 'ignore' });
|
|
105
|
+
child.unref();
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Resolve + run. Returns the OpenCommand used (or null when nothing could be
|
|
110
|
+
* resolved). Throws when the runner reports a launch failure — callers turn
|
|
111
|
+
* that into a friendly die().
|
|
112
|
+
*/
|
|
113
|
+
export function openHit(target: OpenTarget, opts: RunOptions = {}): OpenCommand | null {
|
|
114
|
+
const { runner = defaultOpenRunner, ...resolveOpts } = opts;
|
|
115
|
+
const resolved = resolveOpenCommand(target, resolveOpts);
|
|
116
|
+
if (!resolved) return null;
|
|
117
|
+
runner(resolved.command, resolved.args, { detached: true, stdio: 'ignore' });
|
|
118
|
+
return resolved;
|
|
119
|
+
}
|
package/src/providers.ts
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pluggable embedding providers (issue #124): Ollama and OpenAI-compatible
|
|
3
|
+
* endpoints beside the built-in local transformers.js path.
|
|
4
|
+
*
|
|
5
|
+
* Design: a provider produces (a) an async embedFn with the SAME signature as
|
|
6
|
+
* core.embed(texts, role, model, cacheDir, offline) and (b) a synthetic
|
|
7
|
+
* ModelAdapter descriptor. The descriptor flows through the EXISTING machinery
|
|
8
|
+
* (adapter fingerprint, dim validation, index metadata) so switching providers
|
|
9
|
+
* invalidates vectors exactly like switching local models — no special cases.
|
|
10
|
+
*
|
|
11
|
+
* Local transformers.js stays the zero-config default and the only fully
|
|
12
|
+
* offline path; providers are opt-in via explicit flags.
|
|
13
|
+
*/
|
|
14
|
+
import { loadApiKeyFile } from './serve.js';
|
|
15
|
+
import { ModelAdapter } from './models.js';
|
|
16
|
+
|
|
17
|
+
export type EmbedderRole = 'query' | 'passage';
|
|
18
|
+
export type ExternalEmbedFn = (
|
|
19
|
+
texts: string[],
|
|
20
|
+
role: EmbedderRole,
|
|
21
|
+
model: ModelAdapter,
|
|
22
|
+
cacheDir: string,
|
|
23
|
+
offline: boolean,
|
|
24
|
+
) => Promise<number[][]>;
|
|
25
|
+
|
|
26
|
+
export interface ProviderConfig {
|
|
27
|
+
embedder: 'ollama' | 'openai';
|
|
28
|
+
/** Provider-side model name, e.g. `nomic-embed-text` or `text-embedding-3-small`. */
|
|
29
|
+
model: string;
|
|
30
|
+
baseUrl?: string;
|
|
31
|
+
keyFile?: string;
|
|
32
|
+
/** Injected fetch for tests; defaults to globalThis.fetch. */
|
|
33
|
+
fetchImpl?: typeof fetch;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface ResolvedEmbedder {
|
|
37
|
+
id: string;
|
|
38
|
+
model: string;
|
|
39
|
+
embedFn: ExternalEmbedFn;
|
|
40
|
+
/** Synthetic adapter (probes the endpoint once for dim). Await before indexing. */
|
|
41
|
+
descriptor(): Promise<ModelAdapter>;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function baseOf(cfg: ProviderConfig): string {
|
|
45
|
+
const raw =
|
|
46
|
+
cfg.baseUrl ||
|
|
47
|
+
(cfg.embedder === 'ollama' ? process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' : 'https://api.openai.com/v1');
|
|
48
|
+
return raw.replace(/\/$/, '');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function bearerKey(cfg: ProviderConfig): string | undefined {
|
|
52
|
+
if (cfg.embedder !== 'openai') return undefined;
|
|
53
|
+
if (cfg.keyFile) return loadApiKeyFile(cfg.keyFile);
|
|
54
|
+
return process.env.OPENAI_API_KEY || undefined;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function postJson(fetchImpl: typeof fetch, url: string, headers: Record<string, string>, body: unknown): Promise<any> {
|
|
58
|
+
const res = await fetchImpl(url, {
|
|
59
|
+
method: 'POST',
|
|
60
|
+
headers: { 'content-type': 'application/json', ...headers },
|
|
61
|
+
body: JSON.stringify(body),
|
|
62
|
+
});
|
|
63
|
+
if (!res.ok) {
|
|
64
|
+
const text = await res.text().catch(() => '');
|
|
65
|
+
throw new Error(`embedder ${url} -> HTTP ${res.status}${text ? `: ${text.slice(0, 200)}` : ''}`);
|
|
66
|
+
}
|
|
67
|
+
return res.json();
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Build an external embedder; dim is probed once on first use and cached. */
|
|
71
|
+
export function resolveExternalEmbedder(cfg: ProviderConfig): ResolvedEmbedder {
|
|
72
|
+
const fetchImpl = cfg.fetchImpl ?? globalThis.fetch;
|
|
73
|
+
if (typeof fetchImpl !== 'function') throw new Error('embedder: no fetch implementation available');
|
|
74
|
+
let cachedDim = 0;
|
|
75
|
+
|
|
76
|
+
const embedOneBatch = async (texts: string[]): Promise<number[][]> => {
|
|
77
|
+
if (cfg.embedder === 'ollama') {
|
|
78
|
+
const data = await postJson(fetchImpl, `${baseOf(cfg)}/api/embed`, {}, { model: cfg.model, input: texts });
|
|
79
|
+
const embs = data?.embeddings;
|
|
80
|
+
if (!Array.isArray(embs) || embs.length !== texts.length) {
|
|
81
|
+
throw new Error('ollama /api/embed returned unexpected shape');
|
|
82
|
+
}
|
|
83
|
+
return embs;
|
|
84
|
+
}
|
|
85
|
+
// OpenAI-compatible
|
|
86
|
+
const headers: Record<string, string> = {};
|
|
87
|
+
const key = bearerKey(cfg);
|
|
88
|
+
if (key) headers.authorization = `Bearer ${key}`;
|
|
89
|
+
const data = await postJson(fetchImpl, `${baseOf(cfg)}/embeddings`, headers, {
|
|
90
|
+
model: cfg.model,
|
|
91
|
+
input: texts,
|
|
92
|
+
});
|
|
93
|
+
const rows = data?.data;
|
|
94
|
+
if (!Array.isArray(rows) || rows.length !== texts.length) {
|
|
95
|
+
throw new Error('openai /embeddings returned unexpected shape');
|
|
96
|
+
}
|
|
97
|
+
return rows
|
|
98
|
+
.slice()
|
|
99
|
+
.sort((a: any, b: any) => a.index - b.index)
|
|
100
|
+
.map((r: any) => r.embedding);
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
const embedFn: ExternalEmbedFn = async (texts, _role) => {
|
|
104
|
+
if (texts.length === 0) return [];
|
|
105
|
+
const out = await embedOneBatch(texts);
|
|
106
|
+
if (cachedDim === 0 && out[0]?.length) cachedDim = out[0].length;
|
|
107
|
+
return out;
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
// Dim probing: one tiny request at resolve time keeps index metadata honest
|
|
111
|
+
// without hardcoding per-model tables.
|
|
112
|
+
const descriptorPromise = (async (): Promise<ModelAdapter> => {
|
|
113
|
+
const probe = await embedOneBatch(['dim probe']);
|
|
114
|
+
cachedDim = probe[0]?.length ?? 0;
|
|
115
|
+
if (!cachedDim) throw new Error(`embedder ${cfg.embedder}:${cfg.model} returned an empty embedding`);
|
|
116
|
+
return {
|
|
117
|
+
id: `${cfg.embedder}/${cfg.model}`,
|
|
118
|
+
nativeDim: cachedDim,
|
|
119
|
+
dim: cachedDim,
|
|
120
|
+
queryPrefix: '',
|
|
121
|
+
passagePrefix: '',
|
|
122
|
+
pooling: 'mean',
|
|
123
|
+
normalize: false, // never assume; cosine still works on unnormalized vectors
|
|
124
|
+
family: cfg.embedder,
|
|
125
|
+
};
|
|
126
|
+
})();
|
|
127
|
+
|
|
128
|
+
return {
|
|
129
|
+
id: cfg.embedder,
|
|
130
|
+
model: cfg.model,
|
|
131
|
+
embedFn,
|
|
132
|
+
descriptor: () => descriptorPromise,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Disk-backed persistent query-embedding cache (issue #114).
|
|
3
|
+
*
|
|
4
|
+
* The in-memory QueryEmbeddingCache only helps long-lived processes; every
|
|
5
|
+
* fresh `mdss search` paid full embedding latency. This module adds an L2
|
|
6
|
+
* layer under it: an LRU (512 entries) persisted to `<cacheDir>/query-cache.json`
|
|
7
|
+
* keyed by model@revision|dim|sha256(query) — the same identity the memory
|
|
8
|
+
* cache uses, hashed for compactness.
|
|
9
|
+
*
|
|
10
|
+
* Integrity mirrors vectors.bin: atomic tmp+rename write plus a `.sha256`
|
|
11
|
+
* sidecar. A missing/corrupt pair is silently ignored and rebuilt — a broken
|
|
12
|
+
* cache must never fail a search.
|
|
13
|
+
*/
|
|
14
|
+
import crypto from 'node:crypto';
|
|
15
|
+
import fs from 'node:fs';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
|
|
18
|
+
export const QUERY_CACHE_FILE = 'query-cache.json';
|
|
19
|
+
export const QUERY_CACHE_MAX_ENTRIES = 512;
|
|
20
|
+
|
|
21
|
+
export function buildDiskKey(modelId: string, revision: string, dim: number | undefined, query: string): string {
|
|
22
|
+
const norm = query.trim().toLowerCase();
|
|
23
|
+
return `${modelId}@${revision || 'main'}|${dim ?? 0}|${crypto.createHash('sha256').update(norm).digest('hex')}`;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
interface StoredEntry {
|
|
27
|
+
/** base64 little-endian Float32 payload (encodeVec format). */
|
|
28
|
+
vec: string;
|
|
29
|
+
dim: number;
|
|
30
|
+
ts: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
interface CacheFile {
|
|
34
|
+
version: 1;
|
|
35
|
+
entries: Record<string, StoredEntry>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Per-directory in-process state: loaded entries + dirty flag. */
|
|
39
|
+
const states = new Map<string, { entries: Map<string, StoredEntry>; dirty: boolean; timer?: NodeJS.Timeout }>();
|
|
40
|
+
|
|
41
|
+
function stateFor(cacheDir: string) {
|
|
42
|
+
let st = states.get(cacheDir);
|
|
43
|
+
if (!st) {
|
|
44
|
+
st = { entries: loadFromDisk(cacheDir), dirty: false };
|
|
45
|
+
states.set(cacheDir, st);
|
|
46
|
+
}
|
|
47
|
+
return st;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function filePaths(cacheDir: string) {
|
|
51
|
+
const file = path.join(cacheDir, QUERY_CACHE_FILE);
|
|
52
|
+
return { file, sha: `${file}.sha256`, tmp: `${file}.tmp-${process.pid}` };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function loadFromDisk(cacheDir: string): Map<string, StoredEntry> {
|
|
56
|
+
const out = new Map<string, StoredEntry>();
|
|
57
|
+
const { file, sha } = filePaths(cacheDir);
|
|
58
|
+
let raw: Buffer;
|
|
59
|
+
try {
|
|
60
|
+
raw = fs.readFileSync(file);
|
|
61
|
+
} catch {
|
|
62
|
+
return out;
|
|
63
|
+
}
|
|
64
|
+
try {
|
|
65
|
+
if (fs.existsSync(sha)) {
|
|
66
|
+
const expected = fs.readFileSync(sha, 'utf8').trim().split(/\s+/)[0];
|
|
67
|
+
const actual = crypto.createHash('sha256').update(raw).digest('hex');
|
|
68
|
+
if (!expected || expected !== actual) return out; // corrupt → silent rebuild
|
|
69
|
+
}
|
|
70
|
+
const parsed = JSON.parse(raw.toString('utf8')) as CacheFile;
|
|
71
|
+
if (parsed.version !== 1 || typeof parsed.entries !== 'object') return out;
|
|
72
|
+
for (const [key, e] of Object.entries(parsed.entries)) {
|
|
73
|
+
if (
|
|
74
|
+
typeof key === 'string' &&
|
|
75
|
+
typeof e?.vec === 'string' &&
|
|
76
|
+
Number.isInteger(e?.dim) &&
|
|
77
|
+
(e.dim as number) > 0 &&
|
|
78
|
+
Number.isFinite(e?.ts)
|
|
79
|
+
) {
|
|
80
|
+
out.set(key, { vec: e.vec, dim: e.dim, ts: e.ts });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
} catch {
|
|
84
|
+
return new Map(); // any corruption → treat as empty; next flush overwrites
|
|
85
|
+
}
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function flushSync(cacheDir: string): void {
|
|
90
|
+
const st = states.get(cacheDir);
|
|
91
|
+
if (!st || !st.dirty) return;
|
|
92
|
+
// LRU trim before writing
|
|
93
|
+
const sorted = [...st.entries.entries()].sort((a, b) => b[1].ts - a[1].ts);
|
|
94
|
+
st.entries = new Map(sorted.slice(0, QUERY_CACHE_MAX_ENTRIES));
|
|
95
|
+
const payload: CacheFile = {
|
|
96
|
+
version: 1,
|
|
97
|
+
entries: Object.fromEntries(st.entries),
|
|
98
|
+
};
|
|
99
|
+
const body = Buffer.from(JSON.stringify(payload));
|
|
100
|
+
const digest = crypto.createHash('sha256').update(body).digest('hex');
|
|
101
|
+
const { file, sha, tmp } = filePaths(cacheDir);
|
|
102
|
+
try {
|
|
103
|
+
fs.mkdirSync(cacheDir, { recursive: true });
|
|
104
|
+
fs.writeFileSync(tmp, body);
|
|
105
|
+
fs.renameSync(tmp, file);
|
|
106
|
+
fs.writeFileSync(`${sha}.tmp`, `${digest} ${QUERY_CACHE_FILE}\n`);
|
|
107
|
+
fs.renameSync(`${sha}.tmp`, sha);
|
|
108
|
+
st.dirty = false;
|
|
109
|
+
} catch {
|
|
110
|
+
/* unwritable cacheDir is non-fatal by design */
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function scheduleFlush(cacheDir: string): void {
|
|
115
|
+
const st = stateFor(cacheDir);
|
|
116
|
+
if (st.timer) return;
|
|
117
|
+
st.timer = setTimeout(() => {
|
|
118
|
+
st.timer = undefined;
|
|
119
|
+
flushSync(cacheDir);
|
|
120
|
+
}, 400);
|
|
121
|
+
st.timer.unref?.();
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// best-effort flush on process exit so one-shot CLI runs persist their queries
|
|
125
|
+
let exitHookInstalled = false;
|
|
126
|
+
function ensureExitHook(): void {
|
|
127
|
+
if (exitHookInstalled) return;
|
|
128
|
+
exitHookInstalled = true;
|
|
129
|
+
process.on('exit', () => {
|
|
130
|
+
for (const dir of states.keys()) flushSync(dir);
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Look up a query vector on disk. Returns null on any miss/corruption —
|
|
136
|
+
* callers fall through to embedding without ceremony.
|
|
137
|
+
*/
|
|
138
|
+
export function diskQueryGet(
|
|
139
|
+
cacheDir: string | undefined,
|
|
140
|
+
key: string,
|
|
141
|
+
expectedDim: number | undefined,
|
|
142
|
+
): Float32Array | null {
|
|
143
|
+
if (!cacheDir) return null;
|
|
144
|
+
const st = stateFor(cacheDir);
|
|
145
|
+
const e = st.entries.get(key);
|
|
146
|
+
if (!e) return null;
|
|
147
|
+
try {
|
|
148
|
+
const buf = Buffer.from(e.vec, 'base64');
|
|
149
|
+
if (buf.byteLength !== e.dim * 4) return null;
|
|
150
|
+
const arr = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
|
|
151
|
+
if (expectedDim !== undefined && arr.length !== expectedDim) return null;
|
|
152
|
+
for (const x of arr) if (!Number.isFinite(x)) return null;
|
|
153
|
+
e.ts = Date.now(); // LRU touch
|
|
154
|
+
st.dirty = true;
|
|
155
|
+
scheduleFlush(cacheDir);
|
|
156
|
+
return arr;
|
|
157
|
+
} catch {
|
|
158
|
+
st.entries.delete(key);
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Store a computed query vector (fire-and-forget, debounced flush). */
|
|
164
|
+
export function diskQueryPut(
|
|
165
|
+
cacheDir: string | undefined,
|
|
166
|
+
key: string,
|
|
167
|
+
vec: number[] | Float32Array,
|
|
168
|
+
): void {
|
|
169
|
+
if (!cacheDir) return;
|
|
170
|
+
const f32 = vec instanceof Float32Array ? vec : Float32Array.from(vec);
|
|
171
|
+
const st = stateFor(cacheDir);
|
|
172
|
+
st.entries.set(key, {
|
|
173
|
+
vec: Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength).toString('base64'),
|
|
174
|
+
dim: f32.length,
|
|
175
|
+
ts: Date.now(),
|
|
176
|
+
});
|
|
177
|
+
st.dirty = true;
|
|
178
|
+
ensureExitHook();
|
|
179
|
+
scheduleFlush(cacheDir);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Test helper: force-write now. */
|
|
183
|
+
export function flushQueryCacheNow(cacheDir: string): void {
|
|
184
|
+
flushSync(cacheDir);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Test helper: forget in-process state so the next get/put reloads from disk. */
|
|
188
|
+
export function resetQueryCacheForTest(cacheDir?: string): void {
|
|
189
|
+
if (cacheDir) states.delete(cacheDir);
|
|
190
|
+
else states.clear();
|
|
191
|
+
}
|