md-semantic-search 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +198 -8
  2. package/RESEARCH.md +25 -3
  3. package/bin/cli.mjs +657 -29
  4. package/dist/completions.d.ts +25 -0
  5. package/dist/completions.js +238 -0
  6. package/dist/core.d.ts +31 -0
  7. package/dist/core.js +121 -1
  8. package/dist/download-progress.d.ts +52 -0
  9. package/dist/download-progress.js +112 -0
  10. package/dist/index-format.js +8 -1
  11. package/dist/index.d.ts +6 -4
  12. package/dist/index.js +5 -4
  13. package/dist/indexer.d.ts +1 -0
  14. package/dist/indexer.js +5 -5
  15. package/dist/ivf.d.ts +14 -7
  16. package/dist/ivf.js +126 -18
  17. package/dist/mcp.d.ts +143 -4
  18. package/dist/mcp.js +499 -23
  19. package/dist/models.d.ts +1 -1
  20. package/dist/models.js +52 -31
  21. package/dist/open.d.ts +43 -0
  22. package/dist/open.js +84 -0
  23. package/dist/providers.d.ts +21 -0
  24. package/dist/providers.js +104 -0
  25. package/dist/query-cache-disk.d.ts +14 -0
  26. package/dist/query-cache-disk.js +175 -0
  27. package/dist/rag.d.ts +72 -7
  28. package/dist/rag.js +191 -13
  29. package/dist/repair.d.ts +36 -0
  30. package/dist/repair.js +208 -0
  31. package/dist/rerank.d.ts +20 -4
  32. package/dist/rerank.js +14 -4
  33. package/dist/search.d.ts +35 -0
  34. package/dist/search.js +175 -5
  35. package/dist/serve.d.ts +126 -0
  36. package/dist/serve.js +676 -46
  37. package/dist/tui.d.ts +8 -0
  38. package/dist/tui.js +80 -13
  39. package/dist/watcher.d.ts +16 -0
  40. package/dist/watcher.js +41 -0
  41. package/dist/webui.d.ts +13 -0
  42. package/dist/webui.js +252 -0
  43. package/dist/wikilinks.d.ts +31 -0
  44. package/dist/wikilinks.js +185 -1
  45. package/package.json +1 -1
  46. package/src/completions.ts +255 -0
  47. package/src/core.ts +160 -1
  48. package/src/download-progress.ts +146 -0
  49. package/src/index-format.ts +7 -1
  50. package/src/index.ts +11 -2
  51. package/src/indexer.ts +8 -4
  52. package/src/ivf.ts +157 -20
  53. package/src/mcp.ts +514 -23
  54. package/src/models.ts +58 -33
  55. package/src/open.ts +119 -0
  56. package/src/providers.ts +134 -0
  57. package/src/query-cache-disk.ts +191 -0
  58. package/src/rag.ts +279 -17
  59. package/src/repair.ts +245 -0
  60. package/src/rerank.ts +43 -8
  61. package/src/search.ts +199 -6
  62. package/src/serve.ts +726 -43
  63. package/src/tui.ts +86 -14
  64. package/src/watcher.ts +46 -0
  65. package/src/webui.ts +255 -0
  66. package/src/wikilinks.ts +220 -1
@@ -0,0 +1,25 @@
1
+ /**
2
+ * Shell completions generator (issue #107).
3
+ *
4
+ * Single source of truth: the FLAGS table below drives every shell script AND
5
+ * the drift-guard test (test/completions.test.mjs) that diffs it against the
6
+ * flags actually parsed in bin/cli.mjs. Adding a flag in parseArgs without
7
+ * updating this table fails the test — and vice versa.
8
+ *
9
+ * Static generation only: no runtime dependency, `mdss completions <shell>`
10
+ * prints a self-contained script to stdout.
11
+ */
12
+ export type FlagKind = 'flag' | 'value' | 'file' | 'dir' | 'number';
13
+ export interface FlagSpec {
14
+ /** Primary long form, e.g. "--index-dir". */
15
+ name: string;
16
+ alias?: string;
17
+ kind: FlagKind;
18
+ }
19
+ /** Every option bin/cli.mjs parseArgs understands (issue #107 drift guard). */
20
+ export declare const FLAGS: FlagSpec[];
21
+ export declare const COMMANDS: string[];
22
+ export declare const COMPLETION_SHELLS: readonly ['bash', 'zsh', 'fish', 'powershell'];
23
+ export type CompletionShell = (typeof COMPLETION_SHELLS)[number];
24
+ /** Generate a self-contained completion script; throws on an unknown shell. */
25
+ export declare function generateCompletion(shell: string): string;
@@ -0,0 +1,238 @@
1
+ /**
2
+ * Shell completions generator (issue #107).
3
+ *
4
+ * Single source of truth: the FLAGS table below drives every shell script AND
5
+ * the drift-guard test (test/completions.test.mjs) that diffs it against the
6
+ * flags actually parsed in bin/cli.mjs. Adding a flag in parseArgs without
7
+ * updating this table fails the test — and vice versa.
8
+ *
9
+ * Static generation only: no runtime dependency, `mdss completions <shell>`
10
+ * prints a self-contained script to stdout.
11
+ */
12
+ /** Every option bin/cli.mjs parseArgs understands (issue #107 drift guard). */
13
+ export const FLAGS = [
14
+ // boolean switches
15
+ { name: '--json', kind: 'flag' },
16
+ { name: '--semantic', kind: 'flag' },
17
+ { name: '--offline', kind: 'flag' },
18
+ { name: '--watch', kind: 'flag' },
19
+ { name: '--watch-debug', kind: 'flag' },
20
+ { name: '--no-ui', kind: 'flag' },
21
+ { name: '--mcp', kind: 'flag' },
22
+ { name: '--rerank', kind: 'flag' },
23
+ { name: '--explain', kind: 'flag' },
24
+ { name: '--interactive', alias: '-i', kind: 'flag' },
25
+ { name: '--list-tools', kind: 'flag' },
26
+ { name: '--no-vectors', alias: '--no-vector', kind: 'flag' },
27
+ { name: '--no-query-cache', kind: 'flag' },
28
+ { name: '--recency', kind: 'number' },
29
+ { name: '--embedder', kind: 'value' },
30
+ { name: '--embedder-model', kind: 'value' },
31
+ { name: '--embedder-base-url', kind: 'value' },
32
+ { name: '--embedder-key-file', kind: 'file' },
33
+ { name: '--expand', kind: 'value' },
34
+ { name: '--expand-passages', kind: 'number' },
35
+ { name: '--health-public', kind: 'flag' },
36
+ { name: '--allow-unsecured', kind: 'flag' },
37
+ { name: '--ann', kind: 'flag' },
38
+ { name: '--canonical', kind: 'flag' },
39
+ { name: '--rag', kind: 'flag' },
40
+ { name: '--auto-tag', kind: 'flag' },
41
+ { name: '--auto-summarize', kind: 'flag' },
42
+ { name: '--help', alias: '-h', kind: 'flag' },
43
+ { name: '--version', kind: 'flag' },
44
+ { name: '--open', kind: 'flag' }, // optional N consumed by parseArgs itself
45
+ { name: '--fix', kind: 'flag' },
46
+ { name: '--dry-run', kind: 'flag' },
47
+ // values
48
+ { name: '--workers', kind: 'number' },
49
+ { name: '--config', kind: 'file' },
50
+ { name: '--format', kind: 'value' },
51
+ { name: '--output', alias: '-o', kind: 'file' },
52
+ { name: '--db', kind: 'dir' },
53
+ { name: '--vault', kind: 'dir' },
54
+ { name: '--index-dir', kind: 'dir' },
55
+ { name: '--cache-dir', kind: 'dir' },
56
+ { name: '--model', kind: 'value' },
57
+ { name: '--dimensions', alias: '--dim', kind: 'number' },
58
+ { name: '--k', kind: 'number' },
59
+ { name: '--max-per-file', kind: 'number' },
60
+ { name: '--max-per-doc', kind: 'number' },
61
+ { name: '--target-tokens', kind: 'number' },
62
+ { name: '--port', kind: 'number' },
63
+ { name: '--host', kind: 'value' },
64
+ { name: '--api-key', kind: 'value' },
65
+ { name: '--api-key-file', kind: 'file' },
66
+ { name: '--allowed-host', kind: 'value' },
67
+ { name: '--cors-origin', kind: 'value' },
68
+ { name: '--watch-delay', kind: 'number' },
69
+ { name: '--watch-interval', kind: 'number' },
70
+ { name: '--rate-limit', kind: 'number' },
71
+ { name: '--max-concurrency', kind: 'number' },
72
+ { name: '--since', kind: 'value' },
73
+ { name: '--path', kind: 'value' },
74
+ { name: '--ignore', kind: 'value' },
75
+ { name: '--tag', kind: 'value' },
76
+ { name: '--nprobe', kind: 'number' },
77
+ { name: '--project', kind: 'value' },
78
+ { name: '--type', kind: 'value' },
79
+ { name: '--status', kind: 'value' },
80
+ { name: '--graph-boost', kind: 'number' },
81
+ { name: '--filter', kind: 'value' },
82
+ { name: '--quantize', kind: 'value' },
83
+ { name: '--direction', kind: 'value' },
84
+ { name: '--no-semantic', kind: 'flag' },
85
+ { name: '--session', kind: 'value' },
86
+ { name: '--resume', kind: 'flag' },
87
+ { name: '--llm-endpoint', kind: 'value' },
88
+ { name: '--llm-model', kind: 'value' },
89
+ { name: '--system-prompt', kind: 'value' },
90
+ ];
91
+ export const COMMANDS = [
92
+ 'index', 'stats', 'check', 'doctor', 'export', 'search',
93
+ 'ask', 'chat', 'tui', 'serve', 'mcp', 'models', 'completions', 'related',
94
+ ];
95
+ export const COMPLETION_SHELLS = ['bash', 'zsh', 'fish', 'powershell'];
96
+ const DIR_FLAGS = [...new Set(FLAGS.filter((f) => f.kind === 'dir').flatMap((f) => [f.name, ...(f.alias && !f.alias.startsWith('--') ? [f.alias] : [])]))];
97
+ const FILE_FLAGS = [...new Set(FLAGS.filter((f) => f.kind === 'file').flatMap((f) => [f.name, ...(f.alias && !f.alias.startsWith('--') ? [f.alias] : [])]))];
98
+ const ALL_FLAG_WORDS = FLAGS.flatMap((f) => [f.name, ...(f.alias && f.alias.startsWith('--') ? [f.alias] : [])]);
99
+ function bash() {
100
+ const dirCase = DIR_FLAGS.join('|');
101
+ const fileCase = FILE_FLAGS.join('|');
102
+ return `# bash completion for mdss (md-semantic-search)
103
+ # Generated by \`mdss completions bash\` (issue #107).
104
+ # Install: eval "$(mdss completions bash)"
105
+ _mdss_completion() {
106
+ local cur prev commands flags
107
+ cur="\${COMP_WORDS[COMP_CWORD]}"
108
+ prev="\${COMP_WORDS[COMP_CWORD-1]}"
109
+ commands="${COMMANDS.join(' ')}"
110
+ flags="${ALL_FLAG_WORDS.join(' ')}"
111
+
112
+ case "$prev" in
113
+ ${dirCase})
114
+ mapfile -t COMPREPLY < <(compgen -d -- "$cur"); return 0 ;;
115
+ ${fileCase})
116
+ mapfile -t COMPREPLY < <(compgen -f -- "$cur"); return 0 ;;
117
+ esac
118
+
119
+ if [[ "$cur" == -* ]]; then
120
+ mapfile -t COMPREPLY < <(compgen -W "$flags" -- "$cur")
121
+ return 0
122
+ fi
123
+
124
+ if (( COMP_CWORD <= 2 )); then
125
+ mapfile -t COMPREPLY < <(compgen -W "$commands" -- "$cur")
126
+ fi
127
+ }
128
+ complete -o default -F _mdss_completion mdss md-semantic-search
129
+ `;
130
+ }
131
+ function zsh() {
132
+ const cmdDescs = COMMANDS.map((c) => `'${c}'`).join(' ');
133
+ return `#compdef mdss
134
+ # zsh completion for mdss — generated by \`mdss completions zsh\` (issue #107).
135
+ # Install: mkdir -p ~/.zfunc && mdss completions zsh > ~/.zfunc/_mdss
136
+ # (ensure ~/.zfunc is in fpath BEFORE compinit)
137
+ local -a commands options dir_flags file_flags
138
+ commands=(${cmdDescs})
139
+ options=(${ALL_FLAG_WORDS.map((f) => `'${f}'`).join(' ')})
140
+ dir_flags=(${DIR_FLAGS.map((f) => `'${f}'`).join(' ')})
141
+ file_flags=(${FILE_FLAGS.map((f) => `'${f}'`).join(' ')})
142
+
143
+ _mdss() {
144
+ local cur prev
145
+ cur="\${words[CURRENT]}"
146
+ prev="\${words[CURRENT-1]}"
147
+ if (( CURRENT == 2 )) && [[ "$cur" != -* ]]; then
148
+ _describe -t commands 'mdss command' commands
149
+ return
150
+ fi
151
+ if (( \${dir_flags[(Ie)$prev]} )); then
152
+ _directories
153
+ return
154
+ fi
155
+ if (( \${file_flags[(Ie)$prev]} )); then
156
+ _files
157
+ return
158
+ fi
159
+ if [[ "$cur" == -* ]]; then
160
+ _describe -t options 'mdss option' options
161
+ return
162
+ fi
163
+ }
164
+ compdef _mdss mdss md-semantic-search
165
+ `;
166
+ }
167
+ function fish() {
168
+ const lines = [
169
+ '# fish completion for mdss — generated by `mdss completions fish` (issue #107).',
170
+ '# Install: mdss completions fish > ~/.config/fish/completions/mdss.fish',
171
+ '',
172
+ `complete -c mdss -n '__fish_is_first_arg' -a '${COMMANDS.join(' ')}'`,
173
+ '',
174
+ ];
175
+ const seenShorts = new Set();
176
+ for (const f of FLAGS) {
177
+ const parts = ['complete -c mdss'];
178
+ if (f.alias && !f.alias.startsWith('--') && !seenShorts.has(f.alias)) {
179
+ parts.push(`-s ${f.alias.slice(1)}`);
180
+ seenShorts.add(f.alias);
181
+ }
182
+ parts.push(`-l ${f.name.slice(2)}`);
183
+ if (f.kind !== 'flag')
184
+ parts.push('-r');
185
+ if (f.kind === 'file')
186
+ parts.push('-F');
187
+ if (f.kind === 'dir')
188
+ parts.push('-a "(__fish_complete_directories)"');
189
+ lines.push(parts.join(' '));
190
+ }
191
+ return lines.join('\n') + '\n';
192
+ }
193
+ function powershell() {
194
+ return `# PowerShell completion for mdss — generated by \`mdss completions powershell\` (issue #107).
195
+ # Install: paste into your $PROFILE (or dot-source it from there).
196
+ Register-ArgumentCompleter -CommandName 'mdss','md-semantic-search' -Native -ScriptBlock {
197
+ param($wordToComplete, $commandAst, $cursorPosition)
198
+ $commands = @('${COMMANDS.join("','")}')
199
+ $dirFlags = @('${DIR_FLAGS.join("','")}')
200
+ $fileFlags = @('${FILE_FLAGS.join("','")}')
201
+ $flags = @('${ALL_FLAG_WORDS.join("','")}')
202
+
203
+ $elements = @($commandAst.CommandElements)
204
+ if ($elements.Count -le 1) {
205
+ $commands | Where-Object { $_ -like "$wordToComplete*" } |
206
+ ForEach-Object { [System.Management.Automation.CompletionResult]::new($_, $_, 'Command', $_) }
207
+ return
208
+ }
209
+
210
+ $prev = ''
211
+ if ($elements.Count -ge 2) { $prev = $elements[$elements.Count - 2].ToString() }
212
+
213
+ if (($dirFlags -contains $prev) -or ($fileFlags -contains $prev)) {
214
+ return # defer path completion to PowerShell's native provider
215
+ }
216
+
217
+ if ("$wordToComplete".StartsWith('-')) {
218
+ $flags | Where-Object { $_ -like "$wordToComplete*" } |
219
+ ForEach-Object { [System.Management.Automation.CompletionResult]::new($_, $_, 'ParameterName', $_) }
220
+ return
221
+ }
222
+
223
+ if ($elements.Count -eq 2) {
224
+ $commands | Where-Object { $_ -like "$wordToComplete*" } |
225
+ ForEach-Object { [System.Management.Automation.CompletionResult]::new($_, $_, 'Command', $_) }
226
+ }
227
+ }
228
+ `;
229
+ }
230
+ const GENERATORS = { bash, zsh, fish, powershell };
231
+ /** Generate a self-contained completion script; throws on an unknown shell. */
232
+ export function generateCompletion(shell) {
233
+ const gen = GENERATORS[shell];
234
+ if (!gen) {
235
+ throw new Error(`unknown shell "${shell}" — use one of: ${COMPLETION_SHELLS.join('|')}`);
236
+ }
237
+ return gen();
238
+ }
package/dist/core.d.ts CHANGED
@@ -4,6 +4,11 @@ import { parseFrontmatter, DocumentMetadata } from './frontmatter.js';
4
4
  import { asymmetricCosineInt8 } from './quantization.js';
5
5
  export { PARSER_VERSION, parseFrontmatter, resolveModel, DocumentMetadata, ModelAdapter, Pooling, DType, asymmetricCosineInt8 };
6
6
  export declare const LOCK_FILENAME = ".mdss.lock";
7
+ export type DownloadProgressListener = (e: import('./download-progress.js').DownloadProgressEvent) => void;
8
+ /** Observe model-file download events from every pipeline load (issue #108). Pass null to detach. */
9
+ export declare function setDownloadProgressListener(cb: DownloadProgressListener | null): void;
10
+ /** Low-level emit used by the loaders; exported so tests/tools can simulate events. */
11
+ export declare function emitDownloadEvent(e: import('./download-progress.js').DownloadProgressEvent): void;
7
12
  export declare function pidAlive(pid: number | null): boolean;
8
13
  export interface LockInfo {
9
14
  pid: number;
@@ -43,6 +48,7 @@ export declare function prepareEmbeddingRequest(model: ModelDescriptor, texts: s
43
48
  normalize: boolean;
44
49
  };
45
50
  };
51
+ export declare function truncateAndNormalizeVector(vec: number[] | Float32Array, targetDim: number, normalize?: boolean): number[];
46
52
  export declare function embed(texts: string[], kind: 'query' | 'passage', model: ModelDescriptor, cacheDir: string, offline?: boolean, retryOpts?: {
47
53
  maxRetries?: number;
48
54
  delays?: number[];
@@ -83,3 +89,28 @@ export interface ParsedChunkFile {
83
89
  meta: DocumentMetadata;
84
90
  }
85
91
  export declare function parseFile(absPath: string, dbDir: string, maxChunk?: number, raw?: string): ParsedChunkFile[];
92
+ export interface GetDocLinesOptions {
93
+ file: string;
94
+ fromLine?: number;
95
+ maxLines?: number;
96
+ dbDir?: string;
97
+ chunks?: Array<{
98
+ file: string;
99
+ text: string;
100
+ startLine?: number;
101
+ endLine?: number;
102
+ }>;
103
+ }
104
+ export interface GetDocLinesResult {
105
+ file: string;
106
+ fromLine: number;
107
+ toLine: number;
108
+ lineCount: number;
109
+ totalLines: number;
110
+ text: string;
111
+ }
112
+ /**
113
+ * Line-range retrieval helper (issue #140).
114
+ * Retrieves an exact line span from disk or fallback chunk data with EOF clamping.
115
+ */
116
+ export declare function getDocLines(opts: GetDocLinesOptions): GetDocLinesResult;
package/dist/core.js CHANGED
@@ -13,6 +13,22 @@ export { PARSER_VERSION, parseFrontmatter, resolveModel, asymmetricCosineInt8 };
13
13
  const _extractors = new Map();
14
14
  export const LOCK_FILENAME = '.mdss.lock';
15
15
  const LOCK_STALE_MS = 10 * 60 * 1000;
16
+ let downloadListener = null;
17
+ /** Observe model-file download events from every pipeline load (issue #108). Pass null to detach. */
18
+ export function setDownloadProgressListener(cb) {
19
+ downloadListener = typeof cb === 'function' ? cb : null;
20
+ }
21
+ /** Low-level emit used by the loaders; exported so tests/tools can simulate events. */
22
+ export function emitDownloadEvent(e) {
23
+ if (!downloadListener)
24
+ return;
25
+ try {
26
+ downloadListener(e);
27
+ }
28
+ catch {
29
+ // a broken UI listener must never abort an in-flight model download
30
+ }
31
+ }
16
32
  export function pidAlive(pid) {
17
33
  if (!Number.isInteger(pid) || pid === null || pid <= 0)
18
34
  return false;
@@ -217,6 +233,7 @@ export async function getExtractor(model, cacheDir, offline = false, retryOpts =
217
233
  const pipeOpts = {
218
234
  revision,
219
235
  dtype: model.dtype || 'q8',
236
+ progress_callback: emitDownloadEvent,
220
237
  ...(sessionOptions ? { session_options: sessionOptions } : {}),
221
238
  };
222
239
  const ext = await pipeline('feature-extraction', source, pipeOpts);
@@ -241,11 +258,36 @@ export function prepareEmbeddingRequest(model, texts, kind) {
241
258
  options: { pooling: model.pooling ?? 'mean', normalize: model.normalize !== false },
242
259
  };
243
260
  }
261
+ export function truncateAndNormalizeVector(vec, targetDim, normalize = true) {
262
+ const len = Math.min(vec.length, targetDim);
263
+ const out = new Array(len);
264
+ let norm = 0;
265
+ for (let i = 0; i < len; i++) {
266
+ const val = vec[i];
267
+ out[i] = val;
268
+ norm += val * val;
269
+ }
270
+ if (normalize) {
271
+ norm = Math.sqrt(norm);
272
+ if (norm > 0) {
273
+ for (let i = 0; i < len; i++) {
274
+ out[i] /= norm;
275
+ }
276
+ }
277
+ }
278
+ return out;
279
+ }
244
280
  export async function embed(texts, kind, model, cacheDir, offline = false, retryOpts = {}) {
245
281
  const ext = await getExtractor(model, cacheDir, offline, retryOpts);
246
282
  const { input, options } = prepareEmbeddingRequest(model, texts, kind);
247
283
  const out = await ext(input, options);
248
- return out.tolist();
284
+ const list = out.tolist();
285
+ const targetDim = model.dim;
286
+ if (targetDim && list.length > 0 && list[0].length > targetDim) {
287
+ const doNorm = model.normalize !== false;
288
+ return list.map((vec) => truncateAndNormalizeVector(vec, targetDim, doNorm));
289
+ }
290
+ return list;
249
291
  }
250
292
  export function cosine(a, b, bOffset = 0) {
251
293
  if (b instanceof Int8Array) {
@@ -472,3 +514,81 @@ export function parseFile(absPath, dbDir, maxChunk, raw) {
472
514
  };
473
515
  });
474
516
  }
517
+ /**
518
+ * Line-range retrieval helper (issue #140).
519
+ * Retrieves an exact line span from disk or fallback chunk data with EOF clamping.
520
+ */
521
+ export function getDocLines(opts) {
522
+ const { file, fromLine = 1, maxLines, dbDir, chunks } = opts;
523
+ if (!file || typeof file !== 'string') {
524
+ throw new Error('getDocLines: "file" parameter is required');
525
+ }
526
+ const normalizedFile = file.split(/[/\\]/).join('/');
527
+ let rawContent = null;
528
+ if (dbDir) {
529
+ const candidatePath = path.resolve(dbDir, file);
530
+ try {
531
+ assertSafePath(candidatePath, [dbDir]);
532
+ if (fs.existsSync(candidatePath) && fs.statSync(candidatePath).isFile()) {
533
+ rawContent = fs.readFileSync(candidatePath, 'utf8');
534
+ }
535
+ }
536
+ catch (e) {
537
+ if (e.message?.includes('path traversal guard'))
538
+ throw e;
539
+ }
540
+ }
541
+ if (rawContent === null && Array.isArray(chunks)) {
542
+ const matchingChunks = chunks.filter((c) => c.file === file || c.file === normalizedFile || c.file.split(/[/\\]/).join('/') === normalizedFile);
543
+ if (matchingChunks.length > 0) {
544
+ matchingChunks.sort((a, b) => (a.startLine || 0) - (b.startLine || 0));
545
+ const maxEnd = Math.max(...matchingChunks.map((c) => c.endLine || c.startLine || 1));
546
+ const lineMap = new Array(maxEnd).fill('');
547
+ for (const ch of matchingChunks) {
548
+ const start = (ch.startLine || 1) - 1;
549
+ const chunkLines = (ch.text || '').split(/\r?\n/);
550
+ for (let i = 0; i < chunkLines.length; i++) {
551
+ if (start + i < lineMap.length) {
552
+ lineMap[start + i] = chunkLines[i];
553
+ }
554
+ }
555
+ }
556
+ rawContent = lineMap.join('\n');
557
+ }
558
+ }
559
+ if (rawContent === null) {
560
+ throw new Error(`File not found: ${file}`);
561
+ }
562
+ const lines = rawContent.split(/\r?\n/);
563
+ const totalLines = lines.length;
564
+ let from = typeof fromLine === 'number' && !isNaN(fromLine) ? Math.floor(fromLine) : 1;
565
+ if (from < 1)
566
+ from = 1;
567
+ let selected;
568
+ let to;
569
+ if (totalLines === 0) {
570
+ from = 1;
571
+ to = 0;
572
+ selected = [];
573
+ }
574
+ else if (from > totalLines) {
575
+ from = totalLines;
576
+ to = totalLines;
577
+ selected = [lines[totalLines - 1]];
578
+ }
579
+ else {
580
+ const count = typeof maxLines === 'number' && !isNaN(maxLines) && maxLines > 0
581
+ ? Math.min(Math.floor(maxLines), totalLines - from + 1)
582
+ : totalLines - from + 1;
583
+ to = from + count - 1;
584
+ selected = lines.slice(from - 1, to);
585
+ }
586
+ return {
587
+ file: normalizedFile,
588
+ fromLine: from,
589
+ toLine: to,
590
+ lineCount: selected.length,
591
+ totalLines,
592
+ text: selected.join('\n'),
593
+ };
594
+ }
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Aggregated model-download progress (issue #108).
3
+ *
4
+ * transformers.js emits per-file events (`initiate`/`download`/`progress`/`done`)
5
+ * while a model directory streams in. A model is many files, so a per-file bar
6
+ * flickers uselessly — this aggregator folds them into ONE view: percent across
7
+ * every tracked file, smoothed MB/s, and an ETA over the bytes still missing.
8
+ *
9
+ * The "sawPartial" gate implements the warm-cache acceptance criterion: cached
10
+ * loads surface only instant complete-file events, so unless a real partial
11
+ * transfer was observed the aggregator stays silent.
12
+ */
13
+ export interface DownloadProgressEvent {
14
+ status?: string;
15
+ name?: string;
16
+ file?: string;
17
+ loaded?: number;
18
+ total?: number;
19
+ progress?: number;
20
+ }
21
+ export interface DownloadSnapshot {
22
+ /** True once a partial transfer was observed — gates ALL output. */
23
+ reportable: boolean;
24
+ /** True when every file with a known total has fully arrived. */
25
+ complete: boolean;
26
+ doneFiles: number;
27
+ activeFiles: number;
28
+ loadedBytes: number;
29
+ totalBytes: number;
30
+ /** 0..100 across files with known totals; null while no total is known yet. */
31
+ percent: number | null;
32
+ mbps: number | null;
33
+ etaSec: number | null;
34
+ currentFile: string | null;
35
+ }
36
+ export declare class DownloadProgressAggregator {
37
+ private files;
38
+ private sawPartial;
39
+ private lastSampleAt;
40
+ private lastSampleBytes;
41
+ private mbpsEma;
42
+ private lastRenderAt;
43
+ update(e: DownloadProgressEvent, nowMs?: number): void;
44
+ private sumLoaded;
45
+ snapshot(): DownloadSnapshot;
46
+ /**
47
+ * Render throttle for live views: pass through at most one line per
48
+ * interval, plus ALWAYS the transition into `complete`.
49
+ */
50
+ shouldRender(nowMs: number, minIntervalMs?: number): boolean;
51
+ private renderedComplete;
52
+ }
@@ -0,0 +1,112 @@
1
+ /**
2
+ * Aggregated model-download progress (issue #108).
3
+ *
4
+ * transformers.js emits per-file events (`initiate`/`download`/`progress`/`done`)
5
+ * while a model directory streams in. A model is many files, so a per-file bar
6
+ * flickers uselessly — this aggregator folds them into ONE view: percent across
7
+ * every tracked file, smoothed MB/s, and an ETA over the bytes still missing.
8
+ *
9
+ * The "sawPartial" gate implements the warm-cache acceptance criterion: cached
10
+ * loads surface only instant complete-file events, so unless a real partial
11
+ * transfer was observed the aggregator stays silent.
12
+ */
13
+ const SPEED_SMOOTHING = 0.3; // EMA weight of the newest instantaneous sample
14
+ export class DownloadProgressAggregator {
15
+ files = new Map();
16
+ sawPartial = false;
17
+ lastSampleAt = 0;
18
+ lastSampleBytes = 0;
19
+ mbpsEma = null;
20
+ lastRenderAt = 0;
21
+ update(e, nowMs = Date.now()) {
22
+ if (!e || typeof e.file !== 'string' || e.file.length === 0)
23
+ return;
24
+ let f = this.files.get(e.file);
25
+ if (!f) {
26
+ f = { loaded: 0, total: 0 };
27
+ this.files.set(e.file, f);
28
+ }
29
+ if (typeof e.total === 'number' && Number.isFinite(e.total) && e.total > f.total) {
30
+ f.total = e.total;
31
+ }
32
+ if (typeof e.loaded === 'number' && Number.isFinite(e.loaded) && e.loaded >= 0) {
33
+ // clamp against the known total so a stray event can't overshoot to >100%
34
+ f.loaded = f.total > 0 ? Math.min(e.loaded, f.total) : e.loaded;
35
+ }
36
+ const midTransfer = (e.status === 'download' || e.status === 'progress') && f.total > 0 && f.loaded < f.total;
37
+ if (midTransfer)
38
+ this.sawPartial = true;
39
+ // speed sampling on cumulative byte growth
40
+ const sum = this.sumLoaded();
41
+ if (this.lastSampleAt === 0) {
42
+ this.lastSampleAt = nowMs;
43
+ this.lastSampleBytes = sum;
44
+ }
45
+ else if (sum > this.lastSampleBytes) {
46
+ const dtSec = (nowMs - this.lastSampleAt) / 1000;
47
+ if (dtSec > 0.05) {
48
+ const inst = (sum - this.lastSampleBytes) / dtSec / (1024 * 1024);
49
+ this.mbpsEma = this.mbpsEma === null ? inst : this.mbpsEma * (1 - SPEED_SMOOTHING) + inst * SPEED_SMOOTHING;
50
+ this.lastSampleAt = nowMs;
51
+ this.lastSampleBytes = sum;
52
+ }
53
+ }
54
+ }
55
+ sumLoaded() {
56
+ let sum = 0;
57
+ for (const f of this.files.values())
58
+ sum += f.loaded;
59
+ return sum;
60
+ }
61
+ snapshot() {
62
+ let loadedBytes = 0;
63
+ let totalBytes = 0;
64
+ let doneFiles = 0;
65
+ let activeFiles = 0;
66
+ for (const f of this.files.values()) {
67
+ loadedBytes += f.loaded;
68
+ if (f.total > 0) {
69
+ totalBytes += f.total;
70
+ if (f.loaded >= f.total)
71
+ doneFiles += 1;
72
+ else
73
+ activeFiles += 1;
74
+ }
75
+ }
76
+ const percent = totalBytes > 0 ? Math.min(100, (loadedBytes / totalBytes) * 100) : null;
77
+ const remaining = totalBytes - loadedBytes;
78
+ const etaSec = this.mbpsEma !== null && this.mbpsEma > 0 && remaining > 0 ? remaining / (1024 * 1024) / this.mbpsEma : null;
79
+ return {
80
+ reportable: this.sawPartial,
81
+ complete: totalBytes > 0 && activeFiles === 0,
82
+ doneFiles,
83
+ activeFiles,
84
+ loadedBytes,
85
+ totalBytes,
86
+ percent,
87
+ mbps: this.sawPartial ? this.mbpsEma : null,
88
+ etaSec: this.sawPartial ? etaSec : null,
89
+ currentFile: [...this.files.entries()].find(([, f]) => f.total === 0 || f.loaded < f.total)?.[0] ?? null,
90
+ };
91
+ }
92
+ /**
93
+ * Render throttle for live views: pass through at most one line per
94
+ * interval, plus ALWAYS the transition into `complete`.
95
+ */
96
+ shouldRender(nowMs, minIntervalMs = 400) {
97
+ if (!this.sawPartial)
98
+ return false;
99
+ const justCompleted = this.snapshot().complete && !this.renderedComplete;
100
+ if (justCompleted) {
101
+ this.renderedComplete = true;
102
+ this.lastRenderAt = nowMs;
103
+ return true;
104
+ }
105
+ if (nowMs - this.lastRenderAt >= minIntervalMs) {
106
+ this.lastRenderAt = nowMs;
107
+ return true;
108
+ }
109
+ return false;
110
+ }
111
+ renderedComplete = false;
112
+ }
@@ -90,7 +90,14 @@ export function validateIndexEnvelope(value, location, options = {}) {
90
90
  else if (modelName !== undefined && modelName !== null && typeof modelName !== 'string') {
91
91
  throw new Error(`${location} model metadata must be a string${REBUILD}`);
92
92
  }
93
- const model = resolveModel(modelName);
93
+ let model;
94
+ try {
95
+ const candidateDim = typeof index.dim === 'number' && index.dim > 0 ? index.dim : undefined;
96
+ model = resolveModel(modelName, candidateDim);
97
+ }
98
+ catch {
99
+ model = resolveModel(modelName);
100
+ }
94
101
  const dim = resolveIndexDimension(index.dim, model.dim);
95
102
  if (schema < 3) {
96
103
  return { index, schema, model, dim };
package/dist/index.d.ts CHANGED
@@ -23,15 +23,17 @@ export { buildIndex, chunkHash } from './indexer.js';
23
23
  export { search, searchIndex, loadIndex, tokenize, keywordScores, rrf, QueryEmbeddingCache, } from './search.js';
24
24
  export { resolveModel, MODELS, DEFAULT_MODEL, normalizeAdapter, embeddingAdapterFingerprint, legacyEmbeddingAdapterFingerprint, } from './models.js';
25
25
  export { getReranker, rerankScores, RERANK_MODEL, } from './rerank.js';
26
- export { embed, getExtractor, cosine, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, } from './core.js';
27
- export { MCP_TOOLS, handleMcpRequest, startMcpServer, } from './mcp.js';
26
+ export { embed, getExtractor, cosine, truncateAndNormalizeVector, walkMarkdown, parseFile, chunkMarkdown, splitFrontmatter, extractTitle, globToRegExp, encodeVec, decodeVec, isBinaryIndex, getDocLines, GetDocLinesOptions, GetDocLinesResult, } from './core.js';
27
+ export { MCP_TOOLS, MCP_RESOURCE_TEMPLATES, MCP_PROMPTS, handleMcpRequest, startMcpServer, } from './mcp.js';
28
28
  export { trainIVF, searchIVFCandidates, serializeIVF, deserializeIVF, ANN_THRESHOLD, DEFAULT_NPROBE, } from './ivf.js';
29
- export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, } from './wikilinks.js';
29
+ export { extractLinks, resolveLinks, buildRelationshipGraph, getRelatedNotes, computePageRank, expandGraphNeighborhood, findRelatedNotes, FindRelatedNotesOptions, RelatedNoteHit, RelatedNoteResult, } from './wikilinks.js';
30
30
  export { parseFilter, tokenizeFilter, evaluateFilter, FilterNode, } from './filter.js';
31
31
  export { serializeBinaryIndex, deserializeBinaryIndex, readBinaryHeader, BINARY_MAGIC, BINARY_HEADER_SIZE, } from './binary-format.js';
32
32
  export { quantizeToInt8, dequantizeFromInt8, asymmetricCosineInt8, INT8_SCALE, INT8_INV_SCALE, } from './quantization.js';
33
33
  export { searchFederated, FederatedSearchOptions, FederatedSearchResultHit, VaultConfig, } from './federation.js';
34
34
  export { createFileWatcher, FileWatcher, WatcherOptions, } from './watcher.js';
35
- export { askQuestion, synthesizeAnswer, extractAnswerFallback, AskOptions, SynthesisResult, Citation, DEFAULT_RAG_SYSTEM_PROMPT, } from './rag.js';
35
+ export { askQuestion, synthesizeAnswer, extractAnswerFallback, chatTurn, loadChatSession, saveChatSession, listChatSessions, getSessionsDir, AskOptions, SynthesisResult, Citation, DEFAULT_RAG_SYSTEM_PROMPT, ChatSession, ChatTurn, ChatSourceManifestItem, ChatTurnOptions, } from './rag.js';
36
36
  export { extractKeyphraseTags, summarizePassage, analyzeNote, NoteAnalysis, } from './summarize.js';
37
37
  export { runTui, TuiOptions } from './tui.js';
38
+ export { generateCompletion, FLAGS, COMMANDS, COMPLETION_SHELLS, } from './completions.js';
39
+ export type { FlagSpec, CompletionShell } from './completions.js';