md-semantic-search 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +198 -8
- package/RESEARCH.md +25 -3
- package/bin/cli.mjs +657 -29
- package/dist/completions.d.ts +25 -0
- package/dist/completions.js +238 -0
- package/dist/core.d.ts +31 -0
- package/dist/core.js +121 -1
- package/dist/download-progress.d.ts +52 -0
- package/dist/download-progress.js +112 -0
- package/dist/index-format.js +8 -1
- package/dist/index.d.ts +6 -4
- package/dist/index.js +5 -4
- package/dist/indexer.d.ts +1 -0
- package/dist/indexer.js +5 -5
- package/dist/ivf.d.ts +14 -7
- package/dist/ivf.js +126 -18
- package/dist/mcp.d.ts +143 -4
- package/dist/mcp.js +499 -23
- package/dist/models.d.ts +1 -1
- package/dist/models.js +52 -31
- package/dist/open.d.ts +43 -0
- package/dist/open.js +84 -0
- package/dist/providers.d.ts +21 -0
- package/dist/providers.js +104 -0
- package/dist/query-cache-disk.d.ts +14 -0
- package/dist/query-cache-disk.js +175 -0
- package/dist/rag.d.ts +72 -7
- package/dist/rag.js +191 -13
- package/dist/repair.d.ts +36 -0
- package/dist/repair.js +208 -0
- package/dist/rerank.d.ts +20 -4
- package/dist/rerank.js +14 -4
- package/dist/search.d.ts +35 -0
- package/dist/search.js +175 -5
- package/dist/serve.d.ts +126 -0
- package/dist/serve.js +676 -46
- package/dist/tui.d.ts +8 -0
- package/dist/tui.js +80 -13
- package/dist/watcher.d.ts +16 -0
- package/dist/watcher.js +41 -0
- package/dist/webui.d.ts +13 -0
- package/dist/webui.js +252 -0
- package/dist/wikilinks.d.ts +31 -0
- package/dist/wikilinks.js +185 -1
- package/package.json +1 -1
- package/src/completions.ts +255 -0
- package/src/core.ts +160 -1
- package/src/download-progress.ts +146 -0
- package/src/index-format.ts +7 -1
- package/src/index.ts +11 -2
- package/src/indexer.ts +8 -4
- package/src/ivf.ts +157 -20
- package/src/mcp.ts +514 -23
- package/src/models.ts +58 -33
- package/src/open.ts +119 -0
- package/src/providers.ts +134 -0
- package/src/query-cache-disk.ts +191 -0
- package/src/rag.ts +279 -17
- package/src/repair.ts +245 -0
- package/src/rerank.ts +43 -8
- package/src/search.ts +199 -6
- package/src/serve.ts +726 -43
- package/src/tui.ts +86 -14
- package/src/watcher.ts +46 -0
- package/src/webui.ts +255 -0
- package/src/wikilinks.ts +220 -1
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shell completions generator (issue #107).
|
|
3
|
+
*
|
|
4
|
+
* Single source of truth: the FLAGS table below drives every shell script AND
|
|
5
|
+
* the drift-guard test (test/completions.test.mjs) that diffs it against the
|
|
6
|
+
* flags actually parsed in bin/cli.mjs. Adding a flag in parseArgs without
|
|
7
|
+
* updating this table fails the test — and vice versa.
|
|
8
|
+
*
|
|
9
|
+
* Static generation only: no runtime dependency, `mdss completions <shell>`
|
|
10
|
+
* prints a self-contained script to stdout.
|
|
11
|
+
*/
|
|
12
|
+
export type FlagKind = 'flag' | 'value' | 'file' | 'dir' | 'number';
|
|
13
|
+
export interface FlagSpec {
|
|
14
|
+
/** Primary long form, e.g. "--index-dir". */
|
|
15
|
+
name: string;
|
|
16
|
+
alias?: string;
|
|
17
|
+
kind: FlagKind;
|
|
18
|
+
}
|
|
19
|
+
/** Every option bin/cli.mjs parseArgs understands (issue #107 drift guard). */
|
|
20
|
+
export declare const FLAGS: FlagSpec[];
|
|
21
|
+
export declare const COMMANDS: string[];
|
|
22
|
+
export declare const COMPLETION_SHELLS: readonly ['bash', 'zsh', 'fish', 'powershell'];
|
|
23
|
+
export type CompletionShell = (typeof COMPLETION_SHELLS)[number];
|
|
24
|
+
/** Generate a self-contained completion script; throws on an unknown shell. */
|
|
25
|
+
export declare function generateCompletion(shell: string): string;
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shell completions generator (issue #107).
|
|
3
|
+
*
|
|
4
|
+
* Single source of truth: the FLAGS table below drives every shell script AND
|
|
5
|
+
* the drift-guard test (test/completions.test.mjs) that diffs it against the
|
|
6
|
+
* flags actually parsed in bin/cli.mjs. Adding a flag in parseArgs without
|
|
7
|
+
* updating this table fails the test — and vice versa.
|
|
8
|
+
*
|
|
9
|
+
* Static generation only: no runtime dependency, `mdss completions <shell>`
|
|
10
|
+
* prints a self-contained script to stdout.
|
|
11
|
+
*/
|
|
12
|
+
/** Every option bin/cli.mjs parseArgs understands (issue #107 drift guard). */
|
|
13
|
+
export const FLAGS = [
|
|
14
|
+
// boolean switches
|
|
15
|
+
{ name: '--json', kind: 'flag' },
|
|
16
|
+
{ name: '--semantic', kind: 'flag' },
|
|
17
|
+
{ name: '--offline', kind: 'flag' },
|
|
18
|
+
{ name: '--watch', kind: 'flag' },
|
|
19
|
+
{ name: '--watch-debug', kind: 'flag' },
|
|
20
|
+
{ name: '--no-ui', kind: 'flag' },
|
|
21
|
+
{ name: '--mcp', kind: 'flag' },
|
|
22
|
+
{ name: '--rerank', kind: 'flag' },
|
|
23
|
+
{ name: '--explain', kind: 'flag' },
|
|
24
|
+
{ name: '--interactive', alias: '-i', kind: 'flag' },
|
|
25
|
+
{ name: '--list-tools', kind: 'flag' },
|
|
26
|
+
{ name: '--no-vectors', alias: '--no-vector', kind: 'flag' },
|
|
27
|
+
{ name: '--no-query-cache', kind: 'flag' },
|
|
28
|
+
{ name: '--recency', kind: 'number' },
|
|
29
|
+
{ name: '--embedder', kind: 'value' },
|
|
30
|
+
{ name: '--embedder-model', kind: 'value' },
|
|
31
|
+
{ name: '--embedder-base-url', kind: 'value' },
|
|
32
|
+
{ name: '--embedder-key-file', kind: 'file' },
|
|
33
|
+
{ name: '--expand', kind: 'value' },
|
|
34
|
+
{ name: '--expand-passages', kind: 'number' },
|
|
35
|
+
{ name: '--health-public', kind: 'flag' },
|
|
36
|
+
{ name: '--allow-unsecured', kind: 'flag' },
|
|
37
|
+
{ name: '--ann', kind: 'flag' },
|
|
38
|
+
{ name: '--canonical', kind: 'flag' },
|
|
39
|
+
{ name: '--rag', kind: 'flag' },
|
|
40
|
+
{ name: '--auto-tag', kind: 'flag' },
|
|
41
|
+
{ name: '--auto-summarize', kind: 'flag' },
|
|
42
|
+
{ name: '--help', alias: '-h', kind: 'flag' },
|
|
43
|
+
{ name: '--version', kind: 'flag' },
|
|
44
|
+
{ name: '--open', kind: 'flag' }, // optional N consumed by parseArgs itself
|
|
45
|
+
{ name: '--fix', kind: 'flag' },
|
|
46
|
+
{ name: '--dry-run', kind: 'flag' },
|
|
47
|
+
// values
|
|
48
|
+
{ name: '--workers', kind: 'number' },
|
|
49
|
+
{ name: '--config', kind: 'file' },
|
|
50
|
+
{ name: '--format', kind: 'value' },
|
|
51
|
+
{ name: '--output', alias: '-o', kind: 'file' },
|
|
52
|
+
{ name: '--db', kind: 'dir' },
|
|
53
|
+
{ name: '--vault', kind: 'dir' },
|
|
54
|
+
{ name: '--index-dir', kind: 'dir' },
|
|
55
|
+
{ name: '--cache-dir', kind: 'dir' },
|
|
56
|
+
{ name: '--model', kind: 'value' },
|
|
57
|
+
{ name: '--dimensions', alias: '--dim', kind: 'number' },
|
|
58
|
+
{ name: '--k', kind: 'number' },
|
|
59
|
+
{ name: '--max-per-file', kind: 'number' },
|
|
60
|
+
{ name: '--max-per-doc', kind: 'number' },
|
|
61
|
+
{ name: '--target-tokens', kind: 'number' },
|
|
62
|
+
{ name: '--port', kind: 'number' },
|
|
63
|
+
{ name: '--host', kind: 'value' },
|
|
64
|
+
{ name: '--api-key', kind: 'value' },
|
|
65
|
+
{ name: '--api-key-file', kind: 'file' },
|
|
66
|
+
{ name: '--allowed-host', kind: 'value' },
|
|
67
|
+
{ name: '--cors-origin', kind: 'value' },
|
|
68
|
+
{ name: '--watch-delay', kind: 'number' },
|
|
69
|
+
{ name: '--watch-interval', kind: 'number' },
|
|
70
|
+
{ name: '--rate-limit', kind: 'number' },
|
|
71
|
+
{ name: '--max-concurrency', kind: 'number' },
|
|
72
|
+
{ name: '--since', kind: 'value' },
|
|
73
|
+
{ name: '--path', kind: 'value' },
|
|
74
|
+
{ name: '--ignore', kind: 'value' },
|
|
75
|
+
{ name: '--tag', kind: 'value' },
|
|
76
|
+
{ name: '--nprobe', kind: 'number' },
|
|
77
|
+
{ name: '--project', kind: 'value' },
|
|
78
|
+
{ name: '--type', kind: 'value' },
|
|
79
|
+
{ name: '--status', kind: 'value' },
|
|
80
|
+
{ name: '--graph-boost', kind: 'number' },
|
|
81
|
+
{ name: '--filter', kind: 'value' },
|
|
82
|
+
{ name: '--quantize', kind: 'value' },
|
|
83
|
+
{ name: '--direction', kind: 'value' },
|
|
84
|
+
{ name: '--no-semantic', kind: 'flag' },
|
|
85
|
+
{ name: '--session', kind: 'value' },
|
|
86
|
+
{ name: '--resume', kind: 'flag' },
|
|
87
|
+
{ name: '--llm-endpoint', kind: 'value' },
|
|
88
|
+
{ name: '--llm-model', kind: 'value' },
|
|
89
|
+
{ name: '--system-prompt', kind: 'value' },
|
|
90
|
+
];
|
|
91
|
+
export const COMMANDS = [
|
|
92
|
+
'index', 'stats', 'check', 'doctor', 'export', 'search',
|
|
93
|
+
'ask', 'chat', 'tui', 'serve', 'mcp', 'models', 'completions', 'related',
|
|
94
|
+
];
|
|
95
|
+
export const COMPLETION_SHELLS = ['bash', 'zsh', 'fish', 'powershell'];
|
|
96
|
+
const DIR_FLAGS = [...new Set(FLAGS.filter((f) => f.kind === 'dir').flatMap((f) => [f.name, ...(f.alias && !f.alias.startsWith('--') ? [f.alias] : [])]))];
|
|
97
|
+
const FILE_FLAGS = [...new Set(FLAGS.filter((f) => f.kind === 'file').flatMap((f) => [f.name, ...(f.alias && !f.alias.startsWith('--') ? [f.alias] : [])]))];
|
|
98
|
+
const ALL_FLAG_WORDS = FLAGS.flatMap((f) => [f.name, ...(f.alias && f.alias.startsWith('--') ? [f.alias] : [])]);
|
|
99
|
+
function bash() {
|
|
100
|
+
const dirCase = DIR_FLAGS.join('|');
|
|
101
|
+
const fileCase = FILE_FLAGS.join('|');
|
|
102
|
+
return `# bash completion for mdss (md-semantic-search)
|
|
103
|
+
# Generated by \`mdss completions bash\` (issue #107).
|
|
104
|
+
# Install: eval "$(mdss completions bash)"
|
|
105
|
+
_mdss_completion() {
|
|
106
|
+
local cur prev commands flags
|
|
107
|
+
cur="\${COMP_WORDS[COMP_CWORD]}"
|
|
108
|
+
prev="\${COMP_WORDS[COMP_CWORD-1]}"
|
|
109
|
+
commands="${COMMANDS.join(' ')}"
|
|
110
|
+
flags="${ALL_FLAG_WORDS.join(' ')}"
|
|
111
|
+
|
|
112
|
+
case "$prev" in
|
|
113
|
+
${dirCase})
|
|
114
|
+
mapfile -t COMPREPLY < <(compgen -d -- "$cur"); return 0 ;;
|
|
115
|
+
${fileCase})
|
|
116
|
+
mapfile -t COMPREPLY < <(compgen -f -- "$cur"); return 0 ;;
|
|
117
|
+
esac
|
|
118
|
+
|
|
119
|
+
if [[ "$cur" == -* ]]; then
|
|
120
|
+
mapfile -t COMPREPLY < <(compgen -W "$flags" -- "$cur")
|
|
121
|
+
return 0
|
|
122
|
+
fi
|
|
123
|
+
|
|
124
|
+
if (( COMP_CWORD <= 2 )); then
|
|
125
|
+
mapfile -t COMPREPLY < <(compgen -W "$commands" -- "$cur")
|
|
126
|
+
fi
|
|
127
|
+
}
|
|
128
|
+
complete -o default -F _mdss_completion mdss md-semantic-search
|
|
129
|
+
`;
|
|
130
|
+
}
|
|
131
|
+
function zsh() {
|
|
132
|
+
const cmdDescs = COMMANDS.map((c) => `'${c}'`).join(' ');
|
|
133
|
+
return `#compdef mdss
|
|
134
|
+
# zsh completion for mdss — generated by \`mdss completions zsh\` (issue #107).
|
|
135
|
+
# Install: mkdir -p ~/.zfunc && mdss completions zsh > ~/.zfunc/_mdss
|
|
136
|
+
# (ensure ~/.zfunc is in fpath BEFORE compinit)
|
|
137
|
+
local -a commands options dir_flags file_flags
|
|
138
|
+
commands=(${cmdDescs})
|
|
139
|
+
options=(${ALL_FLAG_WORDS.map((f) => `'${f}'`).join(' ')})
|
|
140
|
+
dir_flags=(${DIR_FLAGS.map((f) => `'${f}'`).join(' ')})
|
|
141
|
+
file_flags=(${FILE_FLAGS.map((f) => `'${f}'`).join(' ')})
|
|
142
|
+
|
|
143
|
+
_mdss() {
|
|
144
|
+
local cur prev
|
|
145
|
+
cur="\${words[CURRENT]}"
|
|
146
|
+
prev="\${words[CURRENT-1]}"
|
|
147
|
+
if (( CURRENT == 2 )) && [[ "$cur" != -* ]]; then
|
|
148
|
+
_describe -t commands 'mdss command' commands
|
|
149
|
+
return
|
|
150
|
+
fi
|
|
151
|
+
if (( \${dir_flags[(Ie)$prev]} )); then
|
|
152
|
+
_directories
|
|
153
|
+
return
|
|
154
|
+
fi
|
|
155
|
+
if (( \${file_flags[(Ie)$prev]} )); then
|
|
156
|
+
_files
|
|
157
|
+
return
|
|
158
|
+
fi
|
|
159
|
+
if [[ "$cur" == -* ]]; then
|
|
160
|
+
_describe -t options 'mdss option' options
|
|
161
|
+
return
|
|
162
|
+
fi
|
|
163
|
+
}
|
|
164
|
+
compdef _mdss mdss md-semantic-search
|
|
165
|
+
`;
|
|
166
|
+
}
|
|
167
|
+
function fish() {
|
|
168
|
+
const lines = [
|
|
169
|
+
'# fish completion for mdss — generated by `mdss completions fish` (issue #107).',
|
|
170
|
+
'# Install: mdss completions fish > ~/.config/fish/completions/mdss.fish',
|
|
171
|
+
'',
|
|
172
|
+
`complete -c mdss -n '__fish_is_first_arg' -a '${COMMANDS.join(' ')}'`,
|
|
173
|
+
'',
|
|
174
|
+
];
|
|
175
|
+
const seenShorts = new Set();
|
|
176
|
+
for (const f of FLAGS) {
|
|
177
|
+
const parts = ['complete -c mdss'];
|
|
178
|
+
if (f.alias && !f.alias.startsWith('--') && !seenShorts.has(f.alias)) {
|
|
179
|
+
parts.push(`-s ${f.alias.slice(1)}`);
|
|
180
|
+
seenShorts.add(f.alias);
|
|
181
|
+
}
|
|
182
|
+
parts.push(`-l ${f.name.slice(2)}`);
|
|
183
|
+
if (f.kind !== 'flag')
|
|
184
|
+
parts.push('-r');
|
|
185
|
+
if (f.kind === 'file')
|
|
186
|
+
parts.push('-F');
|
|
187
|
+
if (f.kind === 'dir')
|
|
188
|
+
parts.push('-a "(__fish_complete_directories)"');
|
|
189
|
+
lines.push(parts.join(' '));
|
|
190
|
+
}
|
|
191
|
+
return lines.join('\n') + '\n';
|
|
192
|
+
}
|
|
193
|
+
function powershell() {
|
|
194
|
+
return `# PowerShell completion for mdss — generated by \`mdss completions powershell\` (issue #107).
|
|
195
|
+
# Install: paste into your $PROFILE (or dot-source it from there).
|
|
196
|
+
Register-ArgumentCompleter -CommandName 'mdss','md-semantic-search' -Native -ScriptBlock {
|
|
197
|
+
param($wordToComplete, $commandAst, $cursorPosition)
|
|
198
|
+
$commands = @('${COMMANDS.join("','")}')
|
|
199
|
+
$dirFlags = @('${DIR_FLAGS.join("','")}')
|
|
200
|
+
$fileFlags = @('${FILE_FLAGS.join("','")}')
|
|
201
|
+
$flags = @('${ALL_FLAG_WORDS.join("','")}')
|
|
202
|
+
|
|
203
|
+
$elements = @($commandAst.CommandElements)
|
|
204
|
+
if ($elements.Count -le 1) {
|
|
205
|
+
$commands | Where-Object { $_ -like "$wordToComplete*" } |
|
|
206
|
+
ForEach-Object { [System.Management.Automation.CompletionResult]::new($_, $_, 'Command', $_) }
|
|
207
|
+
return
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
$prev = ''
|
|
211
|
+
if ($elements.Count -ge 2) { $prev = $elements[$elements.Count - 2].ToString() }
|
|
212
|
+
|
|
213
|
+
if (($dirFlags -contains $prev) -or ($fileFlags -contains $prev)) {
|
|
214
|
+
return # defer path completion to PowerShell's native provider
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if ("$wordToComplete".StartsWith('-')) {
|
|
218
|
+
$flags | Where-Object { $_ -like "$wordToComplete*" } |
|
|
219
|
+
ForEach-Object { [System.Management.Automation.CompletionResult]::new($_, $_, 'ParameterName', $_) }
|
|
220
|
+
return
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
if ($elements.Count -eq 2) {
|
|
224
|
+
$commands | Where-Object { $_ -like "$wordToComplete*" } |
|
|
225
|
+
ForEach-Object { [System.Management.Automation.CompletionResult]::new($_, $_, 'Command', $_) }
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
`;
|
|
229
|
+
}
|
|
230
|
+
const GENERATORS = { bash, zsh, fish, powershell };
|
|
231
|
+
/** Generate a self-contained completion script; throws on an unknown shell. */
|
|
232
|
+
export function generateCompletion(shell) {
|
|
233
|
+
const gen = GENERATORS[shell];
|
|
234
|
+
if (!gen) {
|
|
235
|
+
throw new Error(`unknown shell "${shell}" — use one of: ${COMPLETION_SHELLS.join('|')}`);
|
|
236
|
+
}
|
|
237
|
+
return gen();
|
|
238
|
+
}
|
package/dist/core.d.ts
CHANGED
|
@@ -4,6 +4,11 @@ import { parseFrontmatter, DocumentMetadata } from './frontmatter.js';
|
|
|
4
4
|
import { asymmetricCosineInt8 } from './quantization.js';
|
|
5
5
|
export { PARSER_VERSION, parseFrontmatter, resolveModel, DocumentMetadata, ModelAdapter, Pooling, DType, asymmetricCosineInt8 };
|
|
6
6
|
export declare const LOCK_FILENAME = ".mdss.lock";
|
|
7
|
+
export type DownloadProgressListener = (e: import('./download-progress.js').DownloadProgressEvent) => void;
|
|
8
|
+
/** Observe model-file download events from every pipeline load (issue #108). Pass null to detach. */
|
|
9
|
+
export declare function setDownloadProgressListener(cb: DownloadProgressListener | null): void;
|
|
10
|
+
/** Low-level emit used by the loaders; exported so tests/tools can simulate events. */
|
|
11
|
+
export declare function emitDownloadEvent(e: import('./download-progress.js').DownloadProgressEvent): void;
|
|
7
12
|
export declare function pidAlive(pid: number | null): boolean;
|
|
8
13
|
export interface LockInfo {
|
|
9
14
|
pid: number;
|
|
@@ -43,6 +48,7 @@ export declare function prepareEmbeddingRequest(model: ModelDescriptor, texts: s
|
|
|
43
48
|
normalize: boolean;
|
|
44
49
|
};
|
|
45
50
|
};
|
|
51
|
+
export declare function truncateAndNormalizeVector(vec: number[] | Float32Array, targetDim: number, normalize?: boolean): number[];
|
|
46
52
|
export declare function embed(texts: string[], kind: 'query' | 'passage', model: ModelDescriptor, cacheDir: string, offline?: boolean, retryOpts?: {
|
|
47
53
|
maxRetries?: number;
|
|
48
54
|
delays?: number[];
|
|
@@ -83,3 +89,28 @@ export interface ParsedChunkFile {
|
|
|
83
89
|
meta: DocumentMetadata;
|
|
84
90
|
}
|
|
85
91
|
export declare function parseFile(absPath: string, dbDir: string, maxChunk?: number, raw?: string): ParsedChunkFile[];
|
|
92
|
+
export interface GetDocLinesOptions {
|
|
93
|
+
file: string;
|
|
94
|
+
fromLine?: number;
|
|
95
|
+
maxLines?: number;
|
|
96
|
+
dbDir?: string;
|
|
97
|
+
chunks?: Array<{
|
|
98
|
+
file: string;
|
|
99
|
+
text: string;
|
|
100
|
+
startLine?: number;
|
|
101
|
+
endLine?: number;
|
|
102
|
+
}>;
|
|
103
|
+
}
|
|
104
|
+
export interface GetDocLinesResult {
|
|
105
|
+
file: string;
|
|
106
|
+
fromLine: number;
|
|
107
|
+
toLine: number;
|
|
108
|
+
lineCount: number;
|
|
109
|
+
totalLines: number;
|
|
110
|
+
text: string;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Line-range retrieval helper (issue #140).
|
|
114
|
+
* Retrieves an exact line span from disk or fallback chunk data with EOF clamping.
|
|
115
|
+
*/
|
|
116
|
+
export declare function getDocLines(opts: GetDocLinesOptions): GetDocLinesResult;
|
package/dist/core.js
CHANGED
|
@@ -13,6 +13,22 @@ export { PARSER_VERSION, parseFrontmatter, resolveModel, asymmetricCosineInt8 };
|
|
|
13
13
|
const _extractors = new Map();
|
|
14
14
|
export const LOCK_FILENAME = '.mdss.lock';
|
|
15
15
|
const LOCK_STALE_MS = 10 * 60 * 1000;
|
|
16
|
+
let downloadListener = null;
|
|
17
|
+
/** Observe model-file download events from every pipeline load (issue #108). Pass null to detach. */
|
|
18
|
+
export function setDownloadProgressListener(cb) {
|
|
19
|
+
downloadListener = typeof cb === 'function' ? cb : null;
|
|
20
|
+
}
|
|
21
|
+
/** Low-level emit used by the loaders; exported so tests/tools can simulate events. */
|
|
22
|
+
export function emitDownloadEvent(e) {
|
|
23
|
+
if (!downloadListener)
|
|
24
|
+
return;
|
|
25
|
+
try {
|
|
26
|
+
downloadListener(e);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// a broken UI listener must never abort an in-flight model download
|
|
30
|
+
}
|
|
31
|
+
}
|
|
16
32
|
export function pidAlive(pid) {
|
|
17
33
|
if (!Number.isInteger(pid) || pid === null || pid <= 0)
|
|
18
34
|
return false;
|
|
@@ -217,6 +233,7 @@ export async function getExtractor(model, cacheDir, offline = false, retryOpts =
|
|
|
217
233
|
const pipeOpts = {
|
|
218
234
|
revision,
|
|
219
235
|
dtype: model.dtype || 'q8',
|
|
236
|
+
progress_callback: emitDownloadEvent,
|
|
220
237
|
...(sessionOptions ? { session_options: sessionOptions } : {}),
|
|
221
238
|
};
|
|
222
239
|
const ext = await pipeline('feature-extraction', source, pipeOpts);
|
|
@@ -241,11 +258,36 @@ export function prepareEmbeddingRequest(model, texts, kind) {
|
|
|
241
258
|
options: { pooling: model.pooling ?? 'mean', normalize: model.normalize !== false },
|
|
242
259
|
};
|
|
243
260
|
}
|
|
261
|
+
export function truncateAndNormalizeVector(vec, targetDim, normalize = true) {
|
|
262
|
+
const len = Math.min(vec.length, targetDim);
|
|
263
|
+
const out = new Array(len);
|
|
264
|
+
let norm = 0;
|
|
265
|
+
for (let i = 0; i < len; i++) {
|
|
266
|
+
const val = vec[i];
|
|
267
|
+
out[i] = val;
|
|
268
|
+
norm += val * val;
|
|
269
|
+
}
|
|
270
|
+
if (normalize) {
|
|
271
|
+
norm = Math.sqrt(norm);
|
|
272
|
+
if (norm > 0) {
|
|
273
|
+
for (let i = 0; i < len; i++) {
|
|
274
|
+
out[i] /= norm;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
return out;
|
|
279
|
+
}
|
|
244
280
|
export async function embed(texts, kind, model, cacheDir, offline = false, retryOpts = {}) {
|
|
245
281
|
const ext = await getExtractor(model, cacheDir, offline, retryOpts);
|
|
246
282
|
const { input, options } = prepareEmbeddingRequest(model, texts, kind);
|
|
247
283
|
const out = await ext(input, options);
|
|
248
|
-
|
|
284
|
+
const list = out.tolist();
|
|
285
|
+
const targetDim = model.dim;
|
|
286
|
+
if (targetDim && list.length > 0 && list[0].length > targetDim) {
|
|
287
|
+
const doNorm = model.normalize !== false;
|
|
288
|
+
return list.map((vec) => truncateAndNormalizeVector(vec, targetDim, doNorm));
|
|
289
|
+
}
|
|
290
|
+
return list;
|
|
249
291
|
}
|
|
250
292
|
export function cosine(a, b, bOffset = 0) {
|
|
251
293
|
if (b instanceof Int8Array) {
|
|
@@ -472,3 +514,81 @@ export function parseFile(absPath, dbDir, maxChunk, raw) {
|
|
|
472
514
|
};
|
|
473
515
|
});
|
|
474
516
|
}
|
|
517
|
+
/**
|
|
518
|
+
* Line-range retrieval helper (issue #140).
|
|
519
|
+
* Retrieves an exact line span from disk or fallback chunk data with EOF clamping.
|
|
520
|
+
*/
|
|
521
|
+
export function getDocLines(opts) {
|
|
522
|
+
const { file, fromLine = 1, maxLines, dbDir, chunks } = opts;
|
|
523
|
+
if (!file || typeof file !== 'string') {
|
|
524
|
+
throw new Error('getDocLines: "file" parameter is required');
|
|
525
|
+
}
|
|
526
|
+
const normalizedFile = file.split(/[/\\]/).join('/');
|
|
527
|
+
let rawContent = null;
|
|
528
|
+
if (dbDir) {
|
|
529
|
+
const candidatePath = path.resolve(dbDir, file);
|
|
530
|
+
try {
|
|
531
|
+
assertSafePath(candidatePath, [dbDir]);
|
|
532
|
+
if (fs.existsSync(candidatePath) && fs.statSync(candidatePath).isFile()) {
|
|
533
|
+
rawContent = fs.readFileSync(candidatePath, 'utf8');
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
catch (e) {
|
|
537
|
+
if (e.message?.includes('path traversal guard'))
|
|
538
|
+
throw e;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
if (rawContent === null && Array.isArray(chunks)) {
|
|
542
|
+
const matchingChunks = chunks.filter((c) => c.file === file || c.file === normalizedFile || c.file.split(/[/\\]/).join('/') === normalizedFile);
|
|
543
|
+
if (matchingChunks.length > 0) {
|
|
544
|
+
matchingChunks.sort((a, b) => (a.startLine || 0) - (b.startLine || 0));
|
|
545
|
+
const maxEnd = Math.max(...matchingChunks.map((c) => c.endLine || c.startLine || 1));
|
|
546
|
+
const lineMap = new Array(maxEnd).fill('');
|
|
547
|
+
for (const ch of matchingChunks) {
|
|
548
|
+
const start = (ch.startLine || 1) - 1;
|
|
549
|
+
const chunkLines = (ch.text || '').split(/\r?\n/);
|
|
550
|
+
for (let i = 0; i < chunkLines.length; i++) {
|
|
551
|
+
if (start + i < lineMap.length) {
|
|
552
|
+
lineMap[start + i] = chunkLines[i];
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
rawContent = lineMap.join('\n');
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
if (rawContent === null) {
|
|
560
|
+
throw new Error(`File not found: ${file}`);
|
|
561
|
+
}
|
|
562
|
+
const lines = rawContent.split(/\r?\n/);
|
|
563
|
+
const totalLines = lines.length;
|
|
564
|
+
let from = typeof fromLine === 'number' && !isNaN(fromLine) ? Math.floor(fromLine) : 1;
|
|
565
|
+
if (from < 1)
|
|
566
|
+
from = 1;
|
|
567
|
+
let selected;
|
|
568
|
+
let to;
|
|
569
|
+
if (totalLines === 0) {
|
|
570
|
+
from = 1;
|
|
571
|
+
to = 0;
|
|
572
|
+
selected = [];
|
|
573
|
+
}
|
|
574
|
+
else if (from > totalLines) {
|
|
575
|
+
from = totalLines;
|
|
576
|
+
to = totalLines;
|
|
577
|
+
selected = [lines[totalLines - 1]];
|
|
578
|
+
}
|
|
579
|
+
else {
|
|
580
|
+
const count = typeof maxLines === 'number' && !isNaN(maxLines) && maxLines > 0
|
|
581
|
+
? Math.min(Math.floor(maxLines), totalLines - from + 1)
|
|
582
|
+
: totalLines - from + 1;
|
|
583
|
+
to = from + count - 1;
|
|
584
|
+
selected = lines.slice(from - 1, to);
|
|
585
|
+
}
|
|
586
|
+
return {
|
|
587
|
+
file: normalizedFile,
|
|
588
|
+
fromLine: from,
|
|
589
|
+
toLine: to,
|
|
590
|
+
lineCount: selected.length,
|
|
591
|
+
totalLines,
|
|
592
|
+
text: selected.join('\n'),
|
|
593
|
+
};
|
|
594
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Aggregated model-download progress (issue #108).
|
|
3
|
+
*
|
|
4
|
+
* transformers.js emits per-file events (`initiate`/`download`/`progress`/`done`)
|
|
5
|
+
* while a model directory streams in. A model is many files, so a per-file bar
|
|
6
|
+
* flickers uselessly — this aggregator folds them into ONE view: percent across
|
|
7
|
+
* every tracked file, smoothed MB/s, and an ETA over the bytes still missing.
|
|
8
|
+
*
|
|
9
|
+
* The "sawPartial" gate implements the warm-cache acceptance criterion: cached
|
|
10
|
+
* loads surface only instant complete-file events, so unless a real partial
|
|
11
|
+
* transfer was observed the aggregator stays silent.
|
|
12
|
+
*/
|
|
13
|
+
export interface DownloadProgressEvent {
|
|
14
|
+
status?: string;
|
|
15
|
+
name?: string;
|
|
16
|
+
file?: string;
|
|
17
|
+
loaded?: number;
|
|
18
|
+
total?: number;
|
|
19
|
+
progress?: number;
|
|
20
|
+
}
|
|
21
|
+
export interface DownloadSnapshot {
|
|
22
|
+
/** True once a partial transfer was observed — gates ALL output. */
|
|
23
|
+
reportable: boolean;
|
|
24
|
+
/** True when every file with a known total has fully arrived. */
|
|
25
|
+
complete: boolean;
|
|
26
|
+
doneFiles: number;
|
|
27
|
+
activeFiles: number;
|
|
28
|
+
loadedBytes: number;
|
|
29
|
+
totalBytes: number;
|
|
30
|
+
/** 0..100 across files with known totals; null while no total is known yet. */
|
|
31
|
+
percent: number | null;
|
|
32
|
+
mbps: number | null;
|
|
33
|
+
etaSec: number | null;
|
|
34
|
+
currentFile: string | null;
|
|
35
|
+
}
|
|
36
|
+
export declare class DownloadProgressAggregator {
|
|
37
|
+
private files;
|
|
38
|
+
private sawPartial;
|
|
39
|
+
private lastSampleAt;
|
|
40
|
+
private lastSampleBytes;
|
|
41
|
+
private mbpsEma;
|
|
42
|
+
private lastRenderAt;
|
|
43
|
+
update(e: DownloadProgressEvent, nowMs?: number): void;
|
|
44
|
+
private sumLoaded;
|
|
45
|
+
snapshot(): DownloadSnapshot;
|
|
46
|
+
/**
|
|
47
|
+
* Render throttle for live views: pass through at most one line per
|
|
48
|
+
* interval, plus ALWAYS the transition into `complete`.
|
|
49
|
+
*/
|
|
50
|
+
shouldRender(nowMs: number, minIntervalMs?: number): boolean;
|
|
51
|
+
private renderedComplete;
|
|
52
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Aggregated model-download progress (issue #108).
|
|
3
|
+
*
|
|
4
|
+
* transformers.js emits per-file events (`initiate`/`download`/`progress`/`done`)
|
|
5
|
+
* while a model directory streams in. A model is many files, so a per-file bar
|
|
6
|
+
* flickers uselessly — this aggregator folds them into ONE view: percent across
|
|
7
|
+
* every tracked file, smoothed MB/s, and an ETA over the bytes still missing.
|
|
8
|
+
*
|
|
9
|
+
* The "sawPartial" gate implements the warm-cache acceptance criterion: cached
|
|
10
|
+
* loads surface only instant complete-file events, so unless a real partial
|
|
11
|
+
* transfer was observed the aggregator stays silent.
|
|
12
|
+
*/
|
|
13
|
+
const SPEED_SMOOTHING = 0.3; // EMA weight of the newest instantaneous sample
|
|
14
|
+
export class DownloadProgressAggregator {
|
|
15
|
+
files = new Map();
|
|
16
|
+
sawPartial = false;
|
|
17
|
+
lastSampleAt = 0;
|
|
18
|
+
lastSampleBytes = 0;
|
|
19
|
+
mbpsEma = null;
|
|
20
|
+
lastRenderAt = 0;
|
|
21
|
+
update(e, nowMs = Date.now()) {
|
|
22
|
+
if (!e || typeof e.file !== 'string' || e.file.length === 0)
|
|
23
|
+
return;
|
|
24
|
+
let f = this.files.get(e.file);
|
|
25
|
+
if (!f) {
|
|
26
|
+
f = { loaded: 0, total: 0 };
|
|
27
|
+
this.files.set(e.file, f);
|
|
28
|
+
}
|
|
29
|
+
if (typeof e.total === 'number' && Number.isFinite(e.total) && e.total > f.total) {
|
|
30
|
+
f.total = e.total;
|
|
31
|
+
}
|
|
32
|
+
if (typeof e.loaded === 'number' && Number.isFinite(e.loaded) && e.loaded >= 0) {
|
|
33
|
+
// clamp against the known total so a stray event can't overshoot to >100%
|
|
34
|
+
f.loaded = f.total > 0 ? Math.min(e.loaded, f.total) : e.loaded;
|
|
35
|
+
}
|
|
36
|
+
const midTransfer = (e.status === 'download' || e.status === 'progress') && f.total > 0 && f.loaded < f.total;
|
|
37
|
+
if (midTransfer)
|
|
38
|
+
this.sawPartial = true;
|
|
39
|
+
// speed sampling on cumulative byte growth
|
|
40
|
+
const sum = this.sumLoaded();
|
|
41
|
+
if (this.lastSampleAt === 0) {
|
|
42
|
+
this.lastSampleAt = nowMs;
|
|
43
|
+
this.lastSampleBytes = sum;
|
|
44
|
+
}
|
|
45
|
+
else if (sum > this.lastSampleBytes) {
|
|
46
|
+
const dtSec = (nowMs - this.lastSampleAt) / 1000;
|
|
47
|
+
if (dtSec > 0.05) {
|
|
48
|
+
const inst = (sum - this.lastSampleBytes) / dtSec / (1024 * 1024);
|
|
49
|
+
this.mbpsEma = this.mbpsEma === null ? inst : this.mbpsEma * (1 - SPEED_SMOOTHING) + inst * SPEED_SMOOTHING;
|
|
50
|
+
this.lastSampleAt = nowMs;
|
|
51
|
+
this.lastSampleBytes = sum;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
sumLoaded() {
|
|
56
|
+
let sum = 0;
|
|
57
|
+
for (const f of this.files.values())
|
|
58
|
+
sum += f.loaded;
|
|
59
|
+
return sum;
|
|
60
|
+
}
|
|
61
|
+
snapshot() {
|
|
62
|
+
let loadedBytes = 0;
|
|
63
|
+
let totalBytes = 0;
|
|
64
|
+
let doneFiles = 0;
|
|
65
|
+
let activeFiles = 0;
|
|
66
|
+
for (const f of this.files.values()) {
|
|
67
|
+
loadedBytes += f.loaded;
|
|
68
|
+
if (f.total > 0) {
|
|
69
|
+
totalBytes += f.total;
|
|
70
|
+
if (f.loaded >= f.total)
|
|
71
|
+
doneFiles += 1;
|
|
72
|
+
else
|
|
73
|
+
activeFiles += 1;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const percent = totalBytes > 0 ? Math.min(100, (loadedBytes / totalBytes) * 100) : null;
|
|
77
|
+
const remaining = totalBytes - loadedBytes;
|
|
78
|
+
const etaSec = this.mbpsEma !== null && this.mbpsEma > 0 && remaining > 0 ? remaining / (1024 * 1024) / this.mbpsEma : null;
|
|
79
|
+
return {
|
|
80
|
+
reportable: this.sawPartial,
|
|
81
|
+
complete: totalBytes > 0 && activeFiles === 0,
|
|
82
|
+
doneFiles,
|
|
83
|
+
activeFiles,
|
|
84
|
+
loadedBytes,
|
|
85
|
+
totalBytes,
|
|
86
|
+
percent,
|
|
87
|
+
mbps: this.sawPartial ? this.mbpsEma : null,
|
|
88
|
+
etaSec: this.sawPartial ? etaSec : null,
|
|
89
|
+
currentFile: [...this.files.entries()].find(([, f]) => f.total === 0 || f.loaded < f.total)?.[0] ?? null,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Render throttle for live views: pass through at most one line per
|
|
94
|
+
* interval, plus ALWAYS the transition into `complete`.
|
|
95
|
+
*/
|
|
96
|
+
shouldRender(nowMs, minIntervalMs = 400) {
|
|
97
|
+
if (!this.sawPartial)
|
|
98
|
+
return false;
|
|
99
|
+
const justCompleted = this.snapshot().complete && !this.renderedComplete;
|
|
100
|
+
if (justCompleted) {
|
|
101
|
+
this.renderedComplete = true;
|
|
102
|
+
this.lastRenderAt = nowMs;
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
if (nowMs - this.lastRenderAt >= minIntervalMs) {
|
|
106
|
+
this.lastRenderAt = nowMs;
|
|
107
|
+
return true;
|
|
108
|
+
}
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
renderedComplete = false;
|
|
112
|
+
}
|
package/dist/index-format.js
CHANGED
|
@@ -90,7 +90,14 @@ export function validateIndexEnvelope(value, location, options = {}) {
|
|
|
90
90
|
else if (modelName !== undefined && modelName !== null && typeof modelName !== 'string') {
|
|
91
91
|
throw new Error(`${location} model metadata must be a string${REBUILD}`);
|
|
92
92
|
}
|
|
93
|
-
|
|
93
|
+
let model;
|
|
94
|
+
try {
|
|
95
|
+
const candidateDim = typeof index.dim === 'number' && index.dim > 0 ? index.dim : undefined;
|
|
96
|
+
model = resolveModel(modelName, candidateDim);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
model = resolveModel(modelName);
|
|
100
|
+
}
|
|
94
101
|
const dim = resolveIndexDimension(index.dim, model.dim);
|
|
95
102
|
if (schema < 3) {
|
|
96
103
|
return { index, schema, model, dim };
|
package/dist/index.d.ts
CHANGED
|
@@ -23,15 +23,17 @@ export { buildIndex, chunkHash } from './indexer.js';
|
|
|
23
23
|
export { search, searchIndex, loadIndex, tokenize, keywordScores, rrf, QueryEmbeddingCache, } from './search.js';
|
|
24
24
|
export { resolveModel, MODELS, DEFAULT_MODEL, normalizeAdapter, embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint, } from './models.js';
|
|
25
25
|
export { getReranker, rerankScores, RERANK_MODEL, } from './rerank.js';
|
|
26
|
-
export { embed, getExtractor, cosine, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, } from './core.js';
|
|
27
|
-
export { MCP_TOOLS, handleMcpRequest, startMcpServer, } from './mcp.js';
|
|
26
|
+
export { embed, getExtractor, cosine, truncateAndNormalizeVector, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, getDocLines, GetDocLinesOptions, GetDocLinesResult, } from './core.js';
|
|
27
|
+
export { MCP_TOOLS, MCP_RESOURCE_TEMPLATES, MCP_PROMPTS, handleMcpRequest, startMcpServer, } from './mcp.js';
|
|
28
28
|
export { trainIVF, searchIVFCandidates, serializeIVF, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE, } from './ivf.js';
|
|
29
|
-
export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, } from './wikilinks.js';
|
|
29
|
+
export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, findRelatedNotes, FindRelatedNotesOptions, RelatedNoteHit, RelatedNoteResult, } from './wikilinks.js';
|
|
30
30
|
export { parseFilter, tokenizeFilter, evaluateFilter, FilterNode, } from './filter.js';
|
|
31
31
|
export { serializeBinaryIndex, deserializeBinaryIndex, readBinaryHeader, BINARY_MAGIC, BINARY_HEADER_SIZE, } from './binary-format.js';
|
|
32
32
|
export { quantizeToInt8, dequantizeFromInt8, asymmetricCosineInt8, INT8_SCALE, INT8_INV_SCALE, } from './quantization.js';
|
|
33
33
|
export { searchFederated, FederatedSearchOptions, FederatedSearchResultHit, VaultConfig, } from './federation.js';
|
|
34
34
|
export { createFileWatcher, FileWatcher, WatcherOptions, } from './watcher.js';
|
|
35
|
-
export { askQuestion, synthesizeAnswer, extractAnswerFallback, AskOptions, SynthesisResult, Citation, DEFAULT_RAG_SYSTEM_PROMPT, } from './rag.js';
|
|
35
|
+
export { askQuestion, synthesizeAnswer, extractAnswerFallback, chatTurn, loadChatSession, saveChatSession, listChatSessions, getSessionsDir, AskOptions, SynthesisResult, Citation, DEFAULT_RAG_SYSTEM_PROMPT, ChatSession, ChatTurn, ChatSourceManifestItem, ChatTurnOptions, } from './rag.js';
|
|
36
36
|
export { extractKeyphraseTags, summarizePassage, analyzeNote, NoteAnalysis, } from './summarize.js';
|
|
37
37
|
export { runTui, TuiOptions } from './tui.js';
|
|
38
|
+
export { generateCompletion, FLAGS, COMMANDS, COMPLETION_SHELLS, } from './completions.js';
|
|
39
|
+
export type { FlagSpec, CompletionShell } from './completions.js';
|