md-semantic-search 0.5.0 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +44 -1
  2. package/bin/cli.mjs +144 -30
  3. package/dist/binary-format.d.ts +25 -0
  4. package/dist/binary-format.js +178 -0
  5. package/dist/core.d.ts +3 -2
  6. package/dist/core.js +26 -31
  7. package/dist/federation.d.ts +15 -0
  8. package/dist/federation.js +94 -0
  9. package/dist/filter.d.ts +29 -0
  10. package/dist/filter.js +317 -0
  11. package/dist/index-format.js +1 -1
  12. package/dist/index.d.ts +9 -1
  13. package/dist/index.js +8 -0
  14. package/dist/indexer.d.ts +13 -0
  15. package/dist/indexer.js +21 -7
  16. package/dist/markdown-parser.d.ts +1 -0
  17. package/dist/markdown-parser.js +71 -12
  18. package/dist/mcp.d.ts +54 -6
  19. package/dist/mcp.js +59 -11
  20. package/dist/quantization.d.ts +24 -0
  21. package/dist/quantization.js +85 -0
  22. package/dist/rag.d.ts +49 -0
  23. package/dist/rag.js +128 -0
  24. package/dist/search.d.ts +20 -6
  25. package/dist/search.js +175 -23
  26. package/dist/serve.js +44 -3
  27. package/dist/summarize.d.ts +16 -0
  28. package/dist/summarize.js +82 -0
  29. package/dist/tui.d.ts +1 -0
  30. package/dist/tui.js +36 -13
  31. package/dist/watcher.d.ts +11 -0
  32. package/dist/watcher.js +138 -0
  33. package/dist/wikilinks.d.ts +22 -0
  34. package/dist/wikilinks.js +100 -0
  35. package/package.json +3 -1
  36. package/src/binary-format.ts +216 -0
  37. package/src/core.ts +45 -37
  38. package/src/federation.ts +118 -0
  39. package/src/filter.ts +368 -0
  40. package/src/index-format.ts +1 -1
  41. package/src/index.ts +39 -1
  42. package/src/indexer.ts +42 -10
  43. package/src/markdown-parser.ts +80 -10
  44. package/src/mcp.ts +57 -11
  45. package/src/quantization.ts +93 -0
  46. package/src/rag.ts +189 -0
  47. package/src/search.ts +202 -25
  48. package/src/serve.ts +46 -4
  49. package/src/summarize.ts +98 -0
  50. package/src/tui.ts +36 -12
  51. package/src/watcher.ts +157 -0
  52. package/src/wikilinks.ts +132 -0
package/src/watcher.ts ADDED
@@ -0,0 +1,157 @@
1
+ /**
2
+ * OS-Native File Watcher with Zero-Polling Kernel Events (issue #100).
3
+ *
4
+ * Utilizes OS kernel events (ReadDirectoryChangesW on Windows, FSEvents on macOS, inotify on Linux)
5
+ * via recursive `fs.watch`, with automatic debounce coalescing and graceful fallback to interval polling.
6
+ */
7
+ import fs from 'node:fs';
8
+ import path from 'node:path';
9
+ import { globToRegExp } from './core.js';
10
+
11
+ export interface WatcherOptions {
12
+ debounceMs?: number;
13
+ ignore?: string[];
14
+ log?: (msg: string) => void;
15
+ fallbackIntervalMs?: number;
16
+ }
17
+
18
+ export interface FileWatcher {
19
+ close: () => void;
20
+ isNative: boolean;
21
+ }
22
+
23
+ export function createFileWatcher(
24
+ dbPath: string,
25
+ onChange: () => void | Promise<void>,
26
+ options: WatcherOptions = {}
27
+ ): FileWatcher {
28
+ const {
29
+ debounceMs = 250,
30
+ ignore = [],
31
+ log = () => {},
32
+ fallbackIntervalMs = 50,
33
+ } = options;
34
+
35
+ let closed = false;
36
+ let debounceTimer: NodeJS.Timeout | null = null;
37
+
38
+ const ignoreRegexes = [
39
+ /\.mdss($|[\\/])/,
40
+ /\.git($|[\\/])/,
41
+ /\.cache($|[\\/])/,
42
+ ...ignore.map((g) => globToRegExp(g)),
43
+ ];
44
+
45
+ const shouldIgnore = (filename: string | null | undefined): boolean => {
46
+ if (!filename) return false;
47
+ const normalized = filename.split(path.sep).join('/');
48
+ for (const re of ignoreRegexes) {
49
+ if (re.test(normalized)) return true;
50
+ }
51
+ // Only care about markdown files or directory changes
52
+ if (path.extname(filename) && !/\.(md|markdown)$/i.test(filename)) {
53
+ return true;
54
+ }
55
+ return false;
56
+ };
57
+
58
+ const scheduleChange = () => {
59
+ if (closed) return;
60
+ if (debounceTimer) {
61
+ clearTimeout(debounceTimer);
62
+ }
63
+ debounceTimer = setTimeout(async () => {
64
+ if (closed) return;
65
+ try {
66
+ await onChange();
67
+ } catch (err: any) {
68
+ log(`watcher onChange error: ${err.message}`);
69
+ }
70
+ }, debounceMs);
71
+ debounceTimer.unref?.();
72
+ };
73
+
74
+ // Attempt recursive native fs.watch first
75
+ try {
76
+ const fsWatcher = fs.watch(
77
+ dbPath,
78
+ { recursive: true },
79
+ (_eventType, filename) => {
80
+ if (closed) return;
81
+ if (filename && shouldIgnore(filename)) {
82
+ return;
83
+ }
84
+ scheduleChange();
85
+ }
86
+ );
87
+
88
+ fsWatcher.on('error', (err) => {
89
+ log(`Native watcher error (${err.message}), continuing.`);
90
+ });
91
+
92
+ // unref() lets the event loop exit even if the watcher is still active.
93
+ // Without this, on Node 18/Linux the inotify handle keeps the process
94
+ // alive indefinitely after all tests finish (or after close() is called
95
+ // but the handle hasn't been fully released by the kernel yet).
96
+ fsWatcher.unref?.();
97
+
98
+ return {
99
+ isNative: true,
100
+ close: () => {
101
+ closed = true;
102
+ if (debounceTimer) clearTimeout(debounceTimer);
103
+ try {
104
+ fsWatcher.close();
105
+ } catch {}
106
+ },
107
+ };
108
+ } catch (nativeErr: any) {
109
+ log(`Native recursive watch unavailable (${nativeErr.message}); falling back to interval polling.`);
110
+
111
+ // Polling fallback
112
+ let lastMtime = 0;
113
+ const getDirMtime = (dir: string): number => {
114
+ try {
115
+ let max = 0;
116
+ const entries = fs.readdirSync(dir, { withFileTypes: true });
117
+ for (const entry of entries) {
118
+ if (entry.name.startsWith('.') || entry.name === '.mdss') continue;
119
+ const full = path.join(dir, entry.name);
120
+ if (shouldIgnore(entry.name)) continue;
121
+ if (entry.isDirectory()) {
122
+ const sub = getDirMtime(full);
123
+ if (sub > max) max = sub;
124
+ } else if (/\.(md|markdown)$/i.test(entry.name)) {
125
+ try {
126
+ const fileStat = fs.statSync(full);
127
+ if (fileStat.mtimeMs > max) max = fileStat.mtimeMs;
128
+ } catch {}
129
+ }
130
+ }
131
+ return max;
132
+ } catch {
133
+ return 0;
134
+ }
135
+ };
136
+
137
+ lastMtime = getDirMtime(dbPath);
138
+ const pollInterval = setInterval(() => {
139
+ if (closed) return;
140
+ const currentMtime = getDirMtime(dbPath);
141
+ if (currentMtime > lastMtime) {
142
+ lastMtime = currentMtime;
143
+ scheduleChange();
144
+ }
145
+ }, fallbackIntervalMs);
146
+ pollInterval.unref?.();
147
+
148
+ return {
149
+ isNative: false,
150
+ close: () => {
151
+ closed = true;
152
+ if (debounceTimer) clearTimeout(debounceTimer);
153
+ clearInterval(pollInterval);
154
+ },
155
+ };
156
+ }
157
+ }
package/src/wikilinks.ts CHANGED
@@ -242,3 +242,135 @@ export function getRelatedNotes(
242
242
 
243
243
  return results;
244
244
  }
245
+
246
+ export interface PageRankOptions {
247
+ damping?: number;
248
+ maxIterations?: number;
249
+ tolerance?: number;
250
+ }
251
+
252
+ /**
253
+ * Computes in-memory PageRank centrality scores for all nodes in the RelationshipGraph.
254
+ * Handles dangling nodes (nodes with 0 out-degree) and normalizes scores.
255
+ */
256
+ export function computePageRank(
257
+ graph: RelationshipGraph,
258
+ opts: PageRankOptions = {}
259
+ ): Map<string, number> {
260
+ const damping = opts.damping ?? 0.85;
261
+ const maxIterations = opts.maxIterations ?? 30;
262
+ const tolerance = opts.tolerance ?? 1e-6;
263
+
264
+ const nodes = new Set<string>();
265
+ for (const node of graph.outgoing.keys()) nodes.add(node);
266
+ for (const node of graph.backlinks.keys()) nodes.add(node);
267
+
268
+ const nodeList = Array.from(nodes);
269
+ const n = nodeList.length;
270
+ const rankMap = new Map<string, number>();
271
+ if (n === 0) return rankMap;
272
+ if (n === 1) {
273
+ rankMap.set(nodeList[0], 1.0);
274
+ return rankMap;
275
+ }
276
+
277
+ const outNeighbors = new Map<string, Set<string>>();
278
+ for (const node of nodeList) {
279
+ const edges = graph.outgoing.get(node) || [];
280
+ const targets = new Set<string>();
281
+ for (const e of edges) {
282
+ if (nodes.has(e.file) && e.file !== node) {
283
+ targets.add(e.file);
284
+ }
285
+ }
286
+ outNeighbors.set(node, targets);
287
+ }
288
+
289
+ const inNeighbors = new Map<string, Set<string>>();
290
+ for (const node of nodeList) {
291
+ inNeighbors.set(node, new Set<string>());
292
+ }
293
+ for (const [src, targets] of outNeighbors.entries()) {
294
+ for (const tgt of targets) {
295
+ inNeighbors.get(tgt)?.add(src);
296
+ }
297
+ }
298
+
299
+ let ranks = new Map<string, number>();
300
+ const initialRank = 1.0 / n;
301
+ for (const node of nodeList) {
302
+ ranks.set(node, initialRank);
303
+ }
304
+
305
+ for (let iter = 0; iter < maxIterations; iter++) {
306
+ const nextRanks = new Map<string, number>();
307
+
308
+ let danglingSum = 0;
309
+ for (const node of nodeList) {
310
+ const outDeg = outNeighbors.get(node)?.size || 0;
311
+ if (outDeg === 0) {
312
+ danglingSum += ranks.get(node) || 0;
313
+ }
314
+ }
315
+
316
+ let maxDiff = 0;
317
+ const baseRank = (1.0 - damping) / n + (damping * danglingSum) / n;
318
+
319
+ for (const node of nodeList) {
320
+ let inSum = 0;
321
+ const inNodes = inNeighbors.get(node) || new Set();
322
+ for (const inNode of inNodes) {
323
+ const outDeg = outNeighbors.get(inNode)?.size || 1;
324
+ inSum += (ranks.get(inNode) || 0) / outDeg;
325
+ }
326
+
327
+ const newRank = baseRank + damping * inSum;
328
+ nextRanks.set(node, newRank);
329
+
330
+ const diff = Math.abs(newRank - (ranks.get(node) || 0));
331
+ if (diff > maxDiff) maxDiff = diff;
332
+ }
333
+
334
+ ranks = nextRanks;
335
+ if (maxDiff < tolerance) break;
336
+ }
337
+
338
+ return ranks;
339
+ }
340
+
341
+ export interface GraphExpansionOptions {
342
+ maxDepth?: number;
343
+ decay?: number;
344
+ topSeeds?: number;
345
+ }
346
+
347
+ /**
348
+ * 2-hop neighborhood expansion: propagates relevance scores across graph links.
349
+ */
350
+ export function expandGraphNeighborhood(
351
+ graph: RelationshipGraph,
352
+ seedFiles: Array<{ file: string; score: number }>,
353
+ opts: GraphExpansionOptions = {}
354
+ ): Map<string, number> {
355
+ const maxDepth = opts.maxDepth ?? 2;
356
+ const decay = opts.decay ?? 0.5;
357
+ const topSeeds = opts.topSeeds ?? 10;
358
+
359
+ const propagationScores = new Map<string, number>();
360
+ const sortedSeeds = [...seedFiles].sort((a, b) => b.score - a.score).slice(0, topSeeds);
361
+
362
+ for (const seed of sortedSeeds) {
363
+ if (seed.score <= 0) continue;
364
+ const related = getRelatedNotes(graph, seed.file, { direction: 'both', depth: maxDepth });
365
+ for (const rel of related) {
366
+ const weight = seed.score * Math.pow(decay, rel.distance);
367
+ const current = propagationScores.get(rel.file) || 0;
368
+ if (weight > current) {
369
+ propagationScores.set(rel.file, weight);
370
+ }
371
+ }
372
+ }
373
+
374
+ return propagationScores;
375
+ }
376
+