@monoes/monograph 1.4.1 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/__tests__/storage/stores.test.ts +66 -1
- package/dist/src/storage/fts-store.d.ts +7 -0
- package/dist/src/storage/fts-store.d.ts.map +1 -1
- package/dist/src/storage/fts-store.js +126 -2
- package/dist/src/storage/fts-store.js.map +1 -1
- package/dist/src/watch/watcher.d.ts +2 -0
- package/dist/src/watch/watcher.d.ts.map +1 -1
- package/dist/src/watch/watcher.js +23 -1
- package/dist/src/watch/watcher.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/storage/fts-store.ts +132 -2
- package/src/watch/watcher.ts +23 -1
package/package.json
CHANGED
package/src/storage/fts-store.ts
CHANGED
|
@@ -1,5 +1,67 @@
|
|
|
1
1
|
import type Database from 'better-sqlite3';
|
|
2
2
|
|
|
3
|
+
// ── Intent-to-identifier extraction ──────────────────────────────────────────
|
|
4
|
+
|
|
5
|
+
const STOPWORDS = new Set([
|
|
6
|
+
'a','an','the','is','are','was','were','be','been','being','have','has','had',
|
|
7
|
+
'do','does','did','will','would','shall','should','may','might','must','can',
|
|
8
|
+
'could','am','it','its','i','me','my','we','our','you','your','he','she',
|
|
9
|
+
'they','them','their','this','that','these','those','of','in','to','for',
|
|
10
|
+
'with','on','at','from','by','about','as','into','through','during','before',
|
|
11
|
+
'after','above','below','between','out','off','over','under','again','then',
|
|
12
|
+
'once','here','there','when','where','why','how','all','both','each','every',
|
|
13
|
+
'few','more','most','other','some','such','no','nor','not','only','own',
|
|
14
|
+
'same','so','than','too','very','just','because','but','and','or','if',
|
|
15
|
+
'while','what','which','who','whom','up','down','get','set','add','use',
|
|
16
|
+
'find','show','make','let','want','need','like','look','also','new','old',
|
|
17
|
+
]);
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Extract code-relevant search terms from a natural-language query.
|
|
21
|
+
* Strips stopwords, splits camelCase/PascalCase/snake_case, and strips
|
|
22
|
+
* file extensions (so "CLAUDE.md" also tries "CLAUDE").
|
|
23
|
+
* Returns an empty array when the query is already a bare identifier.
|
|
24
|
+
*/
|
|
25
|
+
export function extractSearchTerms(query: string): string[] {
|
|
26
|
+
const words = query.split(/[\s,;:!?()\[\]{}'"]+/).filter(Boolean);
|
|
27
|
+
// If query is a single token, it's already a bare identifier query — nothing to extract.
|
|
28
|
+
// (We used to also bail out for short, stopword-free multi-token queries under the
|
|
29
|
+
// assumption they were "already an identifier query", but that misfires for queries
|
|
30
|
+
// like "ExtensionBridge keepalive reconnect" — three distinct identifier-ish tokens
|
|
31
|
+
// that FTS5 MATCH ANDs together and fails to find as a single row. Since this function
|
|
32
|
+
// only runs as a fallback after the raw MATCH already returned zero rows, there's no
|
|
33
|
+
// extra cost to always extracting terms for multi-token queries.)
|
|
34
|
+
if (words.length <= 1) return [];
|
|
35
|
+
|
|
36
|
+
const terms = new Set<string>();
|
|
37
|
+
for (const word of words) {
|
|
38
|
+
const lower = word.toLowerCase();
|
|
39
|
+
if (STOPWORDS.has(lower)) continue;
|
|
40
|
+
if (word.length < 3) continue;
|
|
41
|
+
|
|
42
|
+
// Strip file extensions: "CLAUDE.md" → also add "CLAUDE"
|
|
43
|
+
const dotIdx = word.lastIndexOf('.');
|
|
44
|
+
if (dotIdx > 0 && dotIdx < word.length - 1) {
|
|
45
|
+
terms.add(word.slice(0, dotIdx));
|
|
46
|
+
terms.add(word);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Split camelCase/PascalCase: "ExtensionBridge" → "Extension", "Bridge"
|
|
50
|
+
const camelParts = word.replace(/([a-z])([A-Z])/g, '$1 $2')
|
|
51
|
+
.replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
|
|
52
|
+
.split(/[\s_-]+/)
|
|
53
|
+
.filter(p => p.length >= 3 && !STOPWORDS.has(p.toLowerCase()));
|
|
54
|
+
if (camelParts.length > 1) {
|
|
55
|
+
for (const part of camelParts) terms.add(part);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Keep the original word
|
|
59
|
+
terms.add(word);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return Array.from(terms);
|
|
63
|
+
}
|
|
64
|
+
|
|
3
65
|
export interface FtsResult {
|
|
4
66
|
id: string;
|
|
5
67
|
name: string;
|
|
@@ -71,6 +133,63 @@ export function ftsSearch(
|
|
|
71
133
|
// FTS MATCH can throw on malformed queries; fall through to LIKE path
|
|
72
134
|
}
|
|
73
135
|
|
|
136
|
+
// ── Intent-extraction fallback: when the raw query looks like natural language
|
|
137
|
+
// and FTS returned nothing, extract code-relevant tokens and retry FTS. ──────
|
|
138
|
+
if (matchRows.length === 0) {
|
|
139
|
+
const extracted = extractSearchTerms(safeQuery);
|
|
140
|
+
if (extracted.length > 0) {
|
|
141
|
+
// Try each extracted term individually via FTS, then merge by best rank
|
|
142
|
+
const termFtsQuery = extracted.map(quoteFtsTerm).join(' OR ');
|
|
143
|
+
let termSql = `
|
|
144
|
+
SELECT n.id, n.name, n.norm_label, n.file_path, n.label,
|
|
145
|
+
n.start_line, n.end_line, nodes_fts.rank
|
|
146
|
+
FROM nodes_fts
|
|
147
|
+
JOIN nodes n ON n.rowid = nodes_fts.rowid
|
|
148
|
+
WHERE nodes_fts MATCH ?
|
|
149
|
+
`;
|
|
150
|
+
const termParams: unknown[] = [termFtsQuery];
|
|
151
|
+
if (label) {
|
|
152
|
+
termSql += ' AND n.label = ?';
|
|
153
|
+
termParams.push(label);
|
|
154
|
+
}
|
|
155
|
+
termSql += ' ORDER BY nodes_fts.rank LIMIT ?';
|
|
156
|
+
termParams.push(limit * 2);
|
|
157
|
+
try {
|
|
158
|
+
matchRows = db.prepare(termSql).all(...(termParams as [string, ...unknown[]])) as Record<string, unknown>[];
|
|
159
|
+
} catch { /* FTS MATCH can throw on malformed queries */ }
|
|
160
|
+
|
|
161
|
+
// Also try individual LIKE for each extracted term to catch camelCase substrings
|
|
162
|
+
if (matchRows.length < limit) {
|
|
163
|
+
const seenIds = new Set(matchRows.map((r) => r.id as string));
|
|
164
|
+
for (const term of extracted) {
|
|
165
|
+
if (matchRows.length >= limit) break;
|
|
166
|
+
const escapedTerm = term.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
|
|
167
|
+
const pat = `%${escapedTerm}%`;
|
|
168
|
+
let termLikeSql = `
|
|
169
|
+
SELECT n.id, n.name, n.norm_label, n.file_path, n.label,
|
|
170
|
+
n.start_line, n.end_line, 0 AS rank
|
|
171
|
+
FROM nodes n
|
|
172
|
+
WHERE (n.name LIKE ? ESCAPE '\\' OR n.norm_label LIKE ? ESCAPE '\\')
|
|
173
|
+
`;
|
|
174
|
+
const termLikeParams: unknown[] = [pat, pat];
|
|
175
|
+
if (label) {
|
|
176
|
+
termLikeSql += ' AND n.label = ?';
|
|
177
|
+
termLikeParams.push(label);
|
|
178
|
+
}
|
|
179
|
+
termLikeSql += ' LIMIT ?';
|
|
180
|
+
termLikeParams.push(limit);
|
|
181
|
+
const likeRows = db.prepare(termLikeSql).all(...(termLikeParams as [string, ...unknown[]])) as Record<string, unknown>[];
|
|
182
|
+
for (const r of likeRows) {
|
|
183
|
+
if (!seenIds.has(r.id as string)) {
|
|
184
|
+
matchRows.push(r);
|
|
185
|
+
seenIds.add(r.id as string);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
74
193
|
// Run the LIKE fallback whenever MATCH threw (malformed/boolean-keyword query) or
|
|
75
194
|
// returned zero rows — not just for short (≤2 char) queries. Short queries need it
|
|
76
195
|
// because trigram requires ≥3 characters to fire; longer queries need it whenever
|
|
@@ -79,13 +198,24 @@ export function ftsSearch(
|
|
|
79
198
|
if (safeQuery.length <= 2 || matchRows.length === 0) {
|
|
80
199
|
const escapedQuery = safeQuery.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_');
|
|
81
200
|
const likePattern = `%${escapedQuery}%`;
|
|
201
|
+
|
|
202
|
+
// Also try without file extension: "CLAUDE.md" → "CLAUDE"
|
|
203
|
+
const dotIdx = safeQuery.lastIndexOf('.');
|
|
204
|
+
const strippedLikePattern = (dotIdx > 0 && dotIdx < safeQuery.length - 1)
|
|
205
|
+
? `%${safeQuery.slice(0, dotIdx).replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_')}%`
|
|
206
|
+
: null;
|
|
207
|
+
|
|
82
208
|
let likeSql = `
|
|
83
209
|
SELECT n.id, n.name, n.norm_label, n.file_path, n.label,
|
|
84
210
|
n.start_line, n.end_line, 0 AS rank
|
|
85
211
|
FROM nodes n
|
|
86
|
-
WHERE (n.name LIKE ? ESCAPE '\\' OR n.norm_label LIKE ? ESCAPE '\\' OR n.file_path LIKE ? ESCAPE '\\'
|
|
87
|
-
`;
|
|
212
|
+
WHERE (n.name LIKE ? ESCAPE '\\' OR n.norm_label LIKE ? ESCAPE '\\' OR n.file_path LIKE ? ESCAPE '\\'`;
|
|
88
213
|
const likeParams: unknown[] = [likePattern, likePattern, likePattern];
|
|
214
|
+
if (strippedLikePattern) {
|
|
215
|
+
likeSql += ` OR n.name LIKE ? ESCAPE '\\'`;
|
|
216
|
+
likeParams.push(strippedLikePattern);
|
|
217
|
+
}
|
|
218
|
+
likeSql += ')';
|
|
89
219
|
if (label) {
|
|
90
220
|
likeSql += ' AND n.label = ?';
|
|
91
221
|
likeParams.push(label);
|
package/src/watch/watcher.ts
CHANGED
|
@@ -13,6 +13,8 @@ export interface WatchAsyncOptions extends WatcherOptions {
|
|
|
13
13
|
force?: boolean;
|
|
14
14
|
codeOnly?: boolean;
|
|
15
15
|
llmMaxSections?: number;
|
|
16
|
+
/** Auto-stop after this many ms of no file changes. Default 30min. 0 = never. */
|
|
17
|
+
idleTimeoutMs?: number;
|
|
16
18
|
}
|
|
17
19
|
|
|
18
20
|
/** Convenience: start a watcher and trigger buildAsync on every change. Returns stop() fn. */
|
|
@@ -23,11 +25,25 @@ export async function watchAsync(
|
|
|
23
25
|
const { buildAsync } = await import('../pipeline/orchestrator.js');
|
|
24
26
|
const watcher = new MonographWatcher(repoPath, { debounceMs: opts.debounceMs ?? 3000 });
|
|
25
27
|
|
|
28
|
+
// Idle timeout: auto-stop after prolonged inactivity to reclaim resources.
|
|
29
|
+
const idleMs = opts.idleTimeoutMs ?? 30 * 60_000; // default 30min
|
|
30
|
+
let idleTimer: ReturnType<typeof setTimeout> | null = null;
|
|
31
|
+
const resetIdle = (): void => {
|
|
32
|
+
if (idleMs <= 0) return;
|
|
33
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
34
|
+
idleTimer = setTimeout(() => {
|
|
35
|
+
opts.onProgress?.({ phase: 'watch', message: `No changes for ${Math.round(idleMs / 60_000)}min — auto-stopping watcher.` });
|
|
36
|
+
watcher.stop().catch(() => {});
|
|
37
|
+
}, idleMs);
|
|
38
|
+
(idleTimer as { unref?: () => void }).unref?.();
|
|
39
|
+
};
|
|
40
|
+
|
|
26
41
|
// monolean: full rebuild per change-batch, serialized — true incremental rebuild
|
|
27
42
|
// (re-parse only changed files) requires restructuring the phase pipeline.
|
|
28
43
|
let building = false;
|
|
29
44
|
let rerun = false;
|
|
30
45
|
watcher.on('monograph:updated', async (files: string[]) => {
|
|
46
|
+
resetIdle();
|
|
31
47
|
if (building) { rerun = true; return; } // coalesce saves that land mid-build
|
|
32
48
|
building = true;
|
|
33
49
|
try {
|
|
@@ -52,7 +68,13 @@ export async function watchAsync(
|
|
|
52
68
|
});
|
|
53
69
|
|
|
54
70
|
await watcher.start();
|
|
55
|
-
|
|
71
|
+
resetIdle(); // start the idle clock
|
|
72
|
+
return {
|
|
73
|
+
stop: async () => {
|
|
74
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
75
|
+
await watcher.stop();
|
|
76
|
+
},
|
|
77
|
+
};
|
|
56
78
|
}
|
|
57
79
|
|
|
58
80
|
export class MonographWatcher extends EventEmitter {
|