ctxman 3.1.1 → 3.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ctxman/llm-profiles.json +6 -6
- package/README.md +16 -10
- package/bin/cli.js +107 -8
- package/bin/cm-ask.js +64 -6
- package/bin/cm-audit.js +76 -0
- package/bin/cm-diagnose.js +199 -0
- package/bin/cm-focus.js +322 -0
- package/bin/cm-map.js +71 -18
- package/bin/cm-pack.js +36 -5
- package/bin/cm-policy.js +107 -2
- package/bin/cm-pr.js +18 -3
- package/bin/cm-search.js +84 -19
- package/bin/cm-task.js +192 -0
- package/bin/cm-who.js +119 -0
- package/bin/cm-why.js +41 -5
- package/index.d.ts +931 -1
- package/index.js +105 -1
- package/lib/analyzers/token-calculator.js +76 -4
- package/lib/api/mcp/server.js +618 -16
- package/lib/cache/CacheManager.js +161 -47
- package/lib/core/Analyzer.js +61 -5
- package/lib/diagnose/index.js +712 -0
- package/lib/focus/index.js +790 -0
- package/lib/map/call-graph.js +570 -0
- package/lib/map/graph.js +30 -5
- package/lib/map/index.js +12 -4
- package/lib/map/rank.js +45 -7
- package/lib/map/render.js +89 -28
- package/lib/pack/index.js +20 -1
- package/lib/pack/select.js +8 -1
- package/lib/policy/audit.js +3 -1
- package/lib/policy/index.js +5 -0
- package/lib/policy/workspace.js +254 -0
- package/lib/pr/pr-context.js +292 -31
- package/lib/pr/pr-renderer.js +25 -10
- package/lib/rag/EmbeddingProvider.js +139 -2
- package/lib/rag/VectorStoreFactory.js +10 -4
- package/lib/rag/ask.js +186 -17
- package/lib/rag/stores/SqliteVectorStore.js +595 -0
- package/lib/search/hybrid.js +324 -0
- package/lib/search/symbols.js +75 -11
- package/lib/task/index.js +630 -0
- package/lib/utils/file-utils.js +1 -0
- package/lib/utils/multi-repo-manager.js +4 -1
- package/lib/watch/FileWatcher.js +121 -31
- package/lib/watch/IncrementalAnalyzer.js +964 -16
- package/lib/who/index.js +893 -0
- package/lib/why/analyzer-bridge.js +223 -0
- package/lib/why/discussions.js +314 -0
- package/lib/why/index.js +356 -30
- package/lib/why/render.js +154 -9
- package/lib/workspace/index.js +1274 -0
- package/package.json +1 -1
|
@@ -1,55 +1,916 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* IncrementalAnalyzer - Incremental Analysis Module
|
|
3
|
-
* v3.
|
|
2
|
+
* IncrementalAnalyzer - Incremental Analysis & Live RAG/Graph Sync Module
|
|
3
|
+
* v3.2.0 - Watch mode with incremental SqliteVectorStore + Knowledge Graph synchronization
|
|
4
|
+
* (Single-Repo and Cross-Repo Multi-Root Workspace support)
|
|
4
5
|
*
|
|
5
6
|
* Responsibilities:
|
|
6
|
-
* - Perform incremental analysis on file changes
|
|
7
|
-
* -
|
|
8
|
-
* -
|
|
9
|
-
* -
|
|
7
|
+
* - Perform incremental token & method analysis on file changes
|
|
8
|
+
* - Incrementally synchronize file embeddings in SqliteVectorStore (SHA-256 deduplicated)
|
|
9
|
+
* - Incrementally synchronize repository knowledge graph (nodes, methods, import edges, symbolRefs)
|
|
10
|
+
* - Resolve cross-repository import edges (<aliasA>/<file> -> <aliasB>/<file>) in multi-repo mode
|
|
11
|
+
* - Enforce per-repository export policy (.ctxman/policy.json deny rules) and secret redaction on sync
|
|
12
|
+
* - Update aggregate statistics and emit analysis/sync events
|
|
10
13
|
*/
|
|
11
14
|
|
|
15
|
+
import fs from 'fs';
|
|
16
|
+
import crypto from 'crypto';
|
|
12
17
|
import { EventEmitter } from 'events';
|
|
13
18
|
import path from 'path';
|
|
14
19
|
import { Analyzer } from '../core/Analyzer.js';
|
|
20
|
+
import { Scanner } from '../core/Scanner.js';
|
|
21
|
+
import MethodAnalyzer from '../analyzers/method-analyzer.js';
|
|
22
|
+
import FileUtils from '../utils/file-utils.js';
|
|
23
|
+
import { redactSecrets } from '../utils/secret-redactor.js';
|
|
24
|
+
import { languageOf, extractImports, extractDeclarations } from '../map/imports.js';
|
|
25
|
+
import { ImportResolver } from '../map/resolver.js';
|
|
26
|
+
import { signatureAt, DEFAULT_MAX_FILE_BYTES } from '../map/graph.js';
|
|
27
|
+
import { SqliteVectorStore } from '../rag/stores/SqliteVectorStore.js';
|
|
28
|
+
import { AutoEmbeddingProvider } from '../rag/EmbeddingProvider.js';
|
|
29
|
+
import { resolveWorkspaceRepos } from '../workspace/index.js';
|
|
15
30
|
import { getLogger } from '../utils/logger.js';
|
|
16
31
|
|
|
17
32
|
const logger = getLogger('IncrementalAnalyzer');
|
|
18
33
|
|
|
34
|
+
const MINIFIED_LINE_LENGTH = 300;
|
|
35
|
+
const IDENTIFIER = /[A-Za-z_$][\w$]*/g;
|
|
36
|
+
|
|
37
|
+
const toPosix = (p) =>
|
|
38
|
+
String(p ?? '')
|
|
39
|
+
.split(path.sep)
|
|
40
|
+
.join('/')
|
|
41
|
+
.replace(/\\/g, '/');
|
|
42
|
+
|
|
43
|
+
function comparePaths(a, b) {
|
|
44
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
45
|
+
}
|
|
46
|
+
|
|
19
47
|
export class IncrementalAnalyzer extends EventEmitter {
|
|
20
48
|
constructor(options = {}) {
|
|
21
49
|
super();
|
|
22
50
|
|
|
51
|
+
const syncEnabled = Boolean(
|
|
52
|
+
options.sync ||
|
|
53
|
+
options.syncRag ||
|
|
54
|
+
options.syncGraph ||
|
|
55
|
+
options.vectorStore ||
|
|
56
|
+
options.storePath
|
|
57
|
+
);
|
|
58
|
+
|
|
23
59
|
this.options = {
|
|
24
60
|
autoAnalyze: true,
|
|
61
|
+
redactSecrets: options.redactSecrets !== false,
|
|
62
|
+
maxFileBytes: options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES,
|
|
63
|
+
sync: syncEnabled,
|
|
64
|
+
syncRag: options.syncRag ?? syncEnabled,
|
|
65
|
+
syncGraph: options.syncGraph ?? syncEnabled,
|
|
66
|
+
hydrateGraphOnStart: options.hydrateGraphOnStart !== false,
|
|
67
|
+
repos: [],
|
|
25
68
|
...options,
|
|
26
69
|
};
|
|
27
70
|
|
|
71
|
+
this.projectRoot = options.projectRoot ? path.resolve(options.projectRoot) : null;
|
|
72
|
+
this.extraRepos = Array.isArray(this.options.repos)
|
|
73
|
+
? this.options.repos.filter(Boolean).map((r) => path.resolve(r))
|
|
74
|
+
: [];
|
|
75
|
+
this.isMultiRepo = this.extraRepos.length > 0;
|
|
76
|
+
this.repos = [];
|
|
77
|
+
this._refreshWorkspaceRepos();
|
|
78
|
+
|
|
28
79
|
this.analyzer = new Analyzer(options);
|
|
80
|
+
this.methodAnalyzer = new MethodAnalyzer();
|
|
29
81
|
this.fileCache = new Map(); // Cache of file analyses
|
|
30
82
|
this.aggregateStats = null;
|
|
83
|
+
|
|
84
|
+
// Incremental RAG & Knowledge Graph state
|
|
85
|
+
this.vectorStore = options.vectorStore || null;
|
|
86
|
+
this.graphNodes = new Map(); // posixRelativePath (or <alias>/<relPath>) -> node
|
|
87
|
+
this.graphEdges = []; // Array<{from, to, weight}>
|
|
88
|
+
this._graphHydrated = false;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Resolve and cache workspace repositories when projectRoot or extraRepos are configured.
|
|
93
|
+
*/
|
|
94
|
+
_refreshWorkspaceRepos() {
|
|
95
|
+
if (!this.projectRoot) {
|
|
96
|
+
this.repos = [];
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
if (this.extraRepos.length > 0) {
|
|
100
|
+
this.isMultiRepo = true;
|
|
101
|
+
this.repos = resolveWorkspaceRepos([this.projectRoot, ...this.extraRepos]);
|
|
102
|
+
} else {
|
|
103
|
+
this.isMultiRepo = false;
|
|
104
|
+
this.repos = [{ root: this.projectRoot, alias: null }];
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Infer projectRoot from a file change event when not explicitly configured.
|
|
110
|
+
* @param {Object} changeEvent
|
|
111
|
+
* @returns {string|null}
|
|
112
|
+
*/
|
|
113
|
+
_inferProjectRoot(changeEvent) {
|
|
114
|
+
if (this.projectRoot) return this.projectRoot;
|
|
115
|
+
if (changeEvent?.path && changeEvent?.relativePath) {
|
|
116
|
+
const full = path.resolve(changeEvent.path);
|
|
117
|
+
const rel = path.normalize(changeEvent.relativePath);
|
|
118
|
+
if (full.endsWith(rel)) {
|
|
119
|
+
const prefix = full.slice(0, full.length - rel.length).replace(/[/\\]+$/, '');
|
|
120
|
+
if (prefix) {
|
|
121
|
+
this.projectRoot = prefix;
|
|
122
|
+
this._refreshWorkspaceRepos();
|
|
123
|
+
return this.projectRoot;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
this.projectRoot = path.dirname(full);
|
|
127
|
+
this._refreshWorkspaceRepos();
|
|
128
|
+
return this.projectRoot;
|
|
129
|
+
}
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Resolve a file change event to its owning repository, repo-relative POSIX path, and graph node path.
|
|
135
|
+
* @param {Object} changeEvent
|
|
136
|
+
* @returns {{repo: Object|null, repoRoot: string, repoRelPath: string, graphPath: string}}
|
|
137
|
+
*/
|
|
138
|
+
_resolveChangeLocation(changeEvent) {
|
|
139
|
+
const rawRel = toPosix(changeEvent?.relativePath || '');
|
|
140
|
+
const fullPath = changeEvent?.path ? path.resolve(changeEvent.path) : '';
|
|
141
|
+
|
|
142
|
+
if (!this.isMultiRepo || this.repos.length === 0) {
|
|
143
|
+
const root = this.projectRoot || process.cwd();
|
|
144
|
+
const repoRelPath =
|
|
145
|
+
rawRel || (fullPath && root ? toPosix(path.relative(root, fullPath)) : '');
|
|
146
|
+
return {
|
|
147
|
+
repo: this.repos[0] || { root, alias: null },
|
|
148
|
+
repoRoot: root,
|
|
149
|
+
repoRelPath,
|
|
150
|
+
graphPath: repoRelPath,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// 1. Explicit repo metadata from FileWatcher
|
|
155
|
+
if (changeEvent?.repo && changeEvent?.repoRoot) {
|
|
156
|
+
const matched = this.repos.find(
|
|
157
|
+
(r) => r.alias === changeEvent.repo || r.root === changeEvent.repoRoot
|
|
158
|
+
) || {
|
|
159
|
+
root: path.resolve(changeEvent.repoRoot),
|
|
160
|
+
alias: changeEvent.repo,
|
|
161
|
+
};
|
|
162
|
+
const repoRelPath = toPosix(
|
|
163
|
+
changeEvent.repoRelativePath ||
|
|
164
|
+
(fullPath
|
|
165
|
+
? path.relative(matched.root, fullPath)
|
|
166
|
+
: rawRel.replace(`${matched.alias}/`, ''))
|
|
167
|
+
);
|
|
168
|
+
return {
|
|
169
|
+
repo: matched,
|
|
170
|
+
repoRoot: matched.root,
|
|
171
|
+
repoRelPath,
|
|
172
|
+
graphPath: `${matched.alias}/${repoRelPath}`,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// 2. Match by absolute file path against configured repository roots (longest match wins)
|
|
177
|
+
if (fullPath) {
|
|
178
|
+
let bestRepo = null;
|
|
179
|
+
for (const repo of this.repos) {
|
|
180
|
+
const rel = path.relative(repo.root, fullPath);
|
|
181
|
+
if (rel && !rel.startsWith('..') && !path.isAbsolute(rel)) {
|
|
182
|
+
if (!bestRepo || repo.root.length > bestRepo.root.length) {
|
|
183
|
+
bestRepo = repo;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
if (bestRepo) {
|
|
188
|
+
const repoRelPath = toPosix(path.relative(bestRepo.root, fullPath));
|
|
189
|
+
return {
|
|
190
|
+
repo: bestRepo,
|
|
191
|
+
repoRoot: bestRepo.root,
|
|
192
|
+
repoRelPath,
|
|
193
|
+
graphPath: `${bestRepo.alias}/${repoRelPath}`,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// 3. Match by <alias>/<path> prefix in relativePath
|
|
199
|
+
for (const repo of this.repos) {
|
|
200
|
+
if (repo.alias && rawRel.startsWith(`${repo.alias}/`)) {
|
|
201
|
+
const repoRelPath = rawRel.slice(repo.alias.length + 1);
|
|
202
|
+
return {
|
|
203
|
+
repo,
|
|
204
|
+
repoRoot: repo.root,
|
|
205
|
+
repoRelPath,
|
|
206
|
+
graphPath: rawRel,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// Fallback to primary repo
|
|
212
|
+
const primary = this.repos[0];
|
|
213
|
+
const repoRelPath = rawRel;
|
|
214
|
+
return {
|
|
215
|
+
repo: primary,
|
|
216
|
+
repoRoot: primary.root,
|
|
217
|
+
repoRelPath,
|
|
218
|
+
graphPath: primary.alias ? `${primary.alias}/${repoRelPath}` : repoRelPath,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Check whether a graph node path (single-repo relative path or multi-repo `<alias>/<path>`)
|
|
224
|
+
* is denied by its owning repository's export policy.
|
|
225
|
+
* @param {string} graphPath
|
|
226
|
+
* @returns {boolean}
|
|
227
|
+
*/
|
|
228
|
+
isPathDenied(graphPath) {
|
|
229
|
+
const normalized = toPosix(graphPath);
|
|
230
|
+
if (!this.projectRoot) return false;
|
|
231
|
+
|
|
232
|
+
if (this.isMultiRepo && this.repos.length > 0) {
|
|
233
|
+
for (const repo of this.repos) {
|
|
234
|
+
if (repo.alias && normalized.startsWith(`${repo.alias}/`)) {
|
|
235
|
+
const repoRelPath = normalized.slice(repo.alias.length + 1);
|
|
236
|
+
try {
|
|
237
|
+
const scanner = new Scanner(repo.root);
|
|
238
|
+
return scanner.isDenied(repoRelPath);
|
|
239
|
+
} catch {
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
try {
|
|
247
|
+
const scanner = new Scanner(this.projectRoot);
|
|
248
|
+
return scanner.isDenied(normalized);
|
|
249
|
+
} catch {
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Ensure the SqliteVectorStore is initialized and baseline graph state is hydrated if enabled.
|
|
256
|
+
* @param {string|null} [rootOverride]\n * @returns {Promise<boolean>}
|
|
257
|
+
*/
|
|
258
|
+
async ensureSyncReady(rootOverride = null) {
|
|
259
|
+
if (!this.options.syncRag && !this.options.syncGraph) {
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const root = rootOverride || this.projectRoot || process.cwd();
|
|
264
|
+
if (!this.projectRoot && root) {
|
|
265
|
+
this.projectRoot = path.resolve(root);
|
|
266
|
+
this._refreshWorkspaceRepos();
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if (!this.vectorStore) {
|
|
270
|
+
const storePath =
|
|
271
|
+
this.options.storePath || path.join(this.projectRoot, '.ctxman', 'self-knowledge');
|
|
272
|
+
const embedder = this.options.embeddingProvider || new AutoEmbeddingProvider();
|
|
273
|
+
this.vectorStore = new SqliteVectorStore(embedder, storePath);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
if (!this.vectorStore.initialized) {
|
|
277
|
+
await this.vectorStore.init();
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
if (this.options.syncGraph && !this._graphHydrated) {
|
|
281
|
+
this._graphHydrated = true;
|
|
282
|
+
const persisted = await this.vectorStore.getGraph();
|
|
283
|
+
for (const node of persisted.nodes) {
|
|
284
|
+
this.graphNodes.set(node.path, {
|
|
285
|
+
...node,
|
|
286
|
+
identifiers: new Set(),
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
this.graphEdges = persisted.edges;
|
|
290
|
+
|
|
291
|
+
if (this.options.hydrateGraphOnStart && this.projectRoot && fs.existsSync(this.projectRoot)) {
|
|
292
|
+
await this._hydrateGraphFromDisk(this.projectRoot);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
return true;
|
|
31
297
|
}
|
|
32
298
|
|
|
33
299
|
/**
|
|
34
|
-
*
|
|
300
|
+
* Hydrate in-memory graph nodes from existing project files so cross-file import resolution
|
|
301
|
+
* and symbol reference counts have full repository visibility.
|
|
302
|
+
* @param {string} root
|
|
303
|
+
*/
|
|
304
|
+
async _hydrateGraphFromDisk(root) {
|
|
305
|
+
const targets =
|
|
306
|
+
this.isMultiRepo && this.repos.length > 0
|
|
307
|
+
? this.repos
|
|
308
|
+
: [{ root: path.resolve(root), alias: null }];
|
|
309
|
+
|
|
310
|
+
let changed = false;
|
|
311
|
+
for (const repo of targets) {
|
|
312
|
+
let scanned = [];
|
|
313
|
+
try {
|
|
314
|
+
const scanner = new Scanner(repo.root);
|
|
315
|
+
scanned = scanner.scan();
|
|
316
|
+
} catch (_err) {
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
for (const file of scanned) {
|
|
321
|
+
const repoRelPath = toPosix(file.relativePath);
|
|
322
|
+
const graphPath =
|
|
323
|
+
this.isMultiRepo && repo.alias ? `${repo.alias}/${repoRelPath}` : repoRelPath;
|
|
324
|
+
const language = languageOf(repoRelPath);
|
|
325
|
+
if (!language || file.size > this.options.maxFileBytes) continue;
|
|
326
|
+
|
|
327
|
+
let content;
|
|
328
|
+
try {
|
|
329
|
+
content = fs.readFileSync(file.path, 'utf-8');
|
|
330
|
+
} catch {
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const contentHash = crypto.createHash('sha256').update(content).digest('hex');
|
|
335
|
+
const existing = this.graphNodes.get(graphPath);
|
|
336
|
+
if (existing && existing.contentHash === contentHash && existing.identifiers?.size > 0) {
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
const node = this._buildNodeFromContent(
|
|
341
|
+
graphPath,
|
|
342
|
+
file.path,
|
|
343
|
+
content,
|
|
344
|
+
file.size,
|
|
345
|
+
contentHash
|
|
346
|
+
);
|
|
347
|
+
if (node) {
|
|
348
|
+
this.graphNodes.set(graphPath, node);
|
|
349
|
+
changed = true;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
if (changed) {
|
|
355
|
+
this._recomputeGraphEdgesAndSymbols();
|
|
356
|
+
await this.vectorStore.syncGraph({
|
|
357
|
+
nodes: Array.from(this.graphNodes.values()),
|
|
358
|
+
edges: this.graphEdges,
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Perform an initial full-repository (or multi-repo workspace) bootstrap sync of both
|
|
365
|
+
* RAG embeddings and the knowledge graph.
|
|
366
|
+
* @param {string} [root=this.projectRoot]
|
|
367
|
+
* @param {Object} [options]
|
|
368
|
+
* @param {string[]} [options.repos]
|
|
369
|
+
* @returns {Promise<{files: number, ragUpserted: number, ragSkipped: number, graphNodes: number, graphEdges: number, repos?: string[]}>}
|
|
370
|
+
*/
|
|
371
|
+
async bootstrapSync(root = this.projectRoot, options = {}) {
|
|
372
|
+
const targetRoot = path.resolve(root || this.projectRoot || process.cwd());
|
|
373
|
+
this.projectRoot = targetRoot;
|
|
374
|
+
if (Array.isArray(options.repos) && options.repos.length > 0) {
|
|
375
|
+
this.extraRepos = options.repos.filter(Boolean).map((r) => path.resolve(r));
|
|
376
|
+
}
|
|
377
|
+
this._refreshWorkspaceRepos();
|
|
378
|
+
await this.ensureSyncReady(targetRoot);
|
|
379
|
+
|
|
380
|
+
const targets =
|
|
381
|
+
this.isMultiRepo && this.repos.length > 0 ? this.repos : [{ root: targetRoot, alias: null }];
|
|
382
|
+
|
|
383
|
+
let totalFiles = 0;
|
|
384
|
+
let ragUpserted = 0;
|
|
385
|
+
let ragSkipped = 0;
|
|
386
|
+
|
|
387
|
+
this.graphNodes.clear();
|
|
388
|
+
|
|
389
|
+
for (const repo of targets) {
|
|
390
|
+
const scanner = new Scanner(repo.root);
|
|
391
|
+
const scanned = scanner.scan();
|
|
392
|
+
totalFiles += scanned.length;
|
|
393
|
+
|
|
394
|
+
for (const file of scanned) {
|
|
395
|
+
if (file.size > this.options.maxFileBytes) continue;
|
|
396
|
+
const repoRelPath = toPosix(file.relativePath);
|
|
397
|
+
if (scanner.isDenied(repoRelPath)) continue;
|
|
398
|
+
|
|
399
|
+
const graphPath =
|
|
400
|
+
this.isMultiRepo && repo.alias ? `${repo.alias}/${repoRelPath}` : repoRelPath;
|
|
401
|
+
|
|
402
|
+
let content;
|
|
403
|
+
try {
|
|
404
|
+
content = fs.readFileSync(file.path, 'utf-8');
|
|
405
|
+
} catch {
|
|
406
|
+
continue;
|
|
407
|
+
}
|
|
408
|
+
if (content.includes('\0')) continue;
|
|
409
|
+
|
|
410
|
+
const contentHash = crypto.createHash('sha256').update(content).digest('hex');
|
|
411
|
+
|
|
412
|
+
if (this.options.syncRag && FileUtils.isText(file.path)) {
|
|
413
|
+
const textToEmbed = this.options.redactSecrets ? redactSecrets(content).text : content;
|
|
414
|
+
const res = await this.vectorStore.addDocuments([
|
|
415
|
+
{
|
|
416
|
+
id: graphPath,
|
|
417
|
+
text: textToEmbed,
|
|
418
|
+
metadata: {
|
|
419
|
+
source: 'watch-sync',
|
|
420
|
+
sourceId: graphPath,
|
|
421
|
+
path: file.path,
|
|
422
|
+
relativePath: graphPath,
|
|
423
|
+
repo: repo.alias || undefined,
|
|
424
|
+
extension: file.extension,
|
|
425
|
+
filename: file.name,
|
|
426
|
+
size: file.size,
|
|
427
|
+
updatedAt: new Date().toISOString(),
|
|
428
|
+
},
|
|
429
|
+
},
|
|
430
|
+
]);
|
|
431
|
+
ragUpserted += res.upserted;
|
|
432
|
+
ragSkipped += res.skipped;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
if (this.options.syncGraph && languageOf(repoRelPath)) {
|
|
436
|
+
const node = this._buildNodeFromContent(
|
|
437
|
+
graphPath,
|
|
438
|
+
file.path,
|
|
439
|
+
content,
|
|
440
|
+
file.size,
|
|
441
|
+
contentHash
|
|
442
|
+
);
|
|
443
|
+
if (node) {
|
|
444
|
+
this.graphNodes.set(graphPath, node);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
if (this.options.syncGraph) {
|
|
451
|
+
this._recomputeGraphEdgesAndSymbols();
|
|
452
|
+
await this.vectorStore.syncGraph({
|
|
453
|
+
nodes: Array.from(this.graphNodes.values()),
|
|
454
|
+
edges: this.graphEdges,
|
|
455
|
+
replaceAll: true,
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
const summary = {
|
|
460
|
+
files: totalFiles,
|
|
461
|
+
ragUpserted,
|
|
462
|
+
ragSkipped,
|
|
463
|
+
graphNodes: this.graphNodes.size,
|
|
464
|
+
graphEdges: this.graphEdges.length,
|
|
465
|
+
...(this.isMultiRepo ? { repos: this.repos.map((r) => r.alias) } : {}),
|
|
466
|
+
};
|
|
467
|
+
|
|
468
|
+
this.emit('sync:bootstrap', summary);
|
|
469
|
+
return summary;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/**
|
|
473
|
+
* Build a knowledge-graph node from file content.
|
|
474
|
+
* @param {string} relPath - POSIX project-relative path (or <alias>/<relPath> in multi-repo mode)
|
|
475
|
+
* @param {string} fullPath - Absolute path on disk
|
|
476
|
+
* @param {string} content - Raw file text
|
|
477
|
+
* @param {number} sizeBytes
|
|
478
|
+
* @param {string} [contentHash]
|
|
479
|
+
* @returns {Object|null}
|
|
480
|
+
*/
|
|
481
|
+
_buildNodeFromContent(relPath, fullPath, content, sizeBytes, contentHash = null) {
|
|
482
|
+
const language = languageOf(relPath);
|
|
483
|
+
if (!language) return null;
|
|
484
|
+
|
|
485
|
+
const lines = content.split('\n');
|
|
486
|
+
if (content.length > 4096 && content.length / lines.length > MINIFIED_LINE_LENGTH) {
|
|
487
|
+
return null;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
const hash = contentHash || crypto.createHash('sha256').update(content).digest('hex');
|
|
491
|
+
const foundMethods = this.methodAnalyzer
|
|
492
|
+
.extractMethods(content, fullPath)
|
|
493
|
+
.sort((a, b) => a.line - b.line || comparePaths(a.name, b.name));
|
|
494
|
+
|
|
495
|
+
const methods = foundMethods.map((method) => {
|
|
496
|
+
let signature = signatureAt(lines, method.line);
|
|
497
|
+
if (this.options.redactSecrets) {
|
|
498
|
+
signature = redactSecrets(signature).text;
|
|
499
|
+
}
|
|
500
|
+
return { name: method.name, line: method.line, signature };
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
return {
|
|
504
|
+
path: relPath,
|
|
505
|
+
language,
|
|
506
|
+
bytes: sizeBytes ?? Buffer.byteLength(content, 'utf-8'),
|
|
507
|
+
contentHash: hash,
|
|
508
|
+
methods,
|
|
509
|
+
symbolRefs: 0,
|
|
510
|
+
unresolvedImports: 0,
|
|
511
|
+
imports: extractImports(language, content),
|
|
512
|
+
declarations: extractDeclarations(language, content),
|
|
513
|
+
identifiers: new Set(content.match(IDENTIFIER)),
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Resolve cross-repository import specifiers in multi-repo workspace mode.
|
|
519
|
+
* Supports:
|
|
520
|
+
* - `<alias>/<subpath>`, `@<alias>/<subpath>`, `@<scope>/<alias>/<subpath>`
|
|
521
|
+
* - Bare `<alias>`, `@<alias>`, `@<scope>/<alias>` (resolves index or src/index)
|
|
522
|
+
* - Relative specifiers (`../../other-repo/src/foo.js`) crossing into another watched repo
|
|
523
|
+
* @param {Object} fromRepo - { root, alias }
|
|
524
|
+
* @param {string} fromRepoRelPath - POSIX path inside fromRepo
|
|
525
|
+
* @param {string} spec - Import specifier string
|
|
526
|
+
* @param {Map<string, ImportResolver>} resolversByAlias
|
|
527
|
+
* @returns {string|null} Resolved `<targetAlias>/<targetRelPath>` or null
|
|
528
|
+
*/
|
|
529
|
+
_resolveCrossRepoSpec(fromRepo, fromRepoRelPath, spec, resolversByAlias) {
|
|
530
|
+
if (!spec || typeof spec !== 'string') return null;
|
|
531
|
+
const cleanSpec = spec.trim().replace(/\\/g, '/');
|
|
532
|
+
if (!cleanSpec) return null;
|
|
533
|
+
|
|
534
|
+
// 1. Relative specifier crossing out of fromRepo into a sibling repo on disk
|
|
535
|
+
if (cleanSpec.startsWith('.')) {
|
|
536
|
+
const absFromDir = path.dirname(path.join(fromRepo.root, fromRepoRelPath));
|
|
537
|
+
const absTargetBase = path.resolve(absFromDir, cleanSpec);
|
|
538
|
+
for (const targetRepo of this.repos) {
|
|
539
|
+
if (targetRepo.alias === fromRepo.alias) continue;
|
|
540
|
+
const relInTarget = path.relative(targetRepo.root, absTargetBase);
|
|
541
|
+
if (relInTarget && !relInTarget.startsWith('..') && !path.isAbsolute(relInTarget)) {
|
|
542
|
+
const targetResolver = resolversByAlias.get(targetRepo.alias);
|
|
543
|
+
if (!targetResolver) continue;
|
|
544
|
+
const resolvedRel = targetResolver.resolveJs('', `./${toPosix(relInTarget)}`);
|
|
545
|
+
if (resolvedRel) {
|
|
546
|
+
return `${targetRepo.alias}/${resolvedRel}`;
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
return null;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// 2. Alias-prefixed specifier: `<alias>/...`, `@<alias>/...`, `@<scope>/<alias>/...`
|
|
554
|
+
const normalized = cleanSpec.replace(/^@[^/]+\//, (match) => {
|
|
555
|
+
const bare = match.slice(1, -1);
|
|
556
|
+
// If `@<alias>/...` directly matches a repo alias, keep `<alias>/...`
|
|
557
|
+
if (this.repos.some((r) => r.alias === bare)) {
|
|
558
|
+
return `${bare}/`;
|
|
559
|
+
}
|
|
560
|
+
// Otherwise strip `@<scope>/` so `@org/<alias>/...` becomes `<alias>/...`
|
|
561
|
+
return '';
|
|
562
|
+
});
|
|
563
|
+
const bareAt = cleanSpec.startsWith('@') ? cleanSpec.slice(1) : cleanSpec;
|
|
564
|
+
|
|
565
|
+
for (const targetRepo of this.repos) {
|
|
566
|
+
if (!targetRepo.alias || targetRepo.alias === fromRepo.alias) continue;
|
|
567
|
+
const alias = targetRepo.alias;
|
|
568
|
+
const targetResolver = resolversByAlias.get(alias);
|
|
569
|
+
if (!targetResolver) continue;
|
|
570
|
+
|
|
571
|
+
let subpath = null;
|
|
572
|
+
if (normalized === alias || bareAt === alias) {
|
|
573
|
+
subpath = 'index';
|
|
574
|
+
} else if (normalized.startsWith(`${alias}/`)) {
|
|
575
|
+
subpath = normalized.slice(alias.length + 1);
|
|
576
|
+
} else if (bareAt.startsWith(`${alias}/`)) {
|
|
577
|
+
subpath = bareAt.slice(alias.length + 1);
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
if (subpath) {
|
|
581
|
+
const candidates = [`./${subpath}`, `./src/${subpath}`, `./lib/${subpath}`];
|
|
582
|
+
for (const cand of candidates) {
|
|
583
|
+
const resolvedRel = targetResolver.resolveJs('', cand);
|
|
584
|
+
if (resolvedRel) {
|
|
585
|
+
return `${alias}/${resolvedRel}`;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
return null;
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/**
|
|
595
|
+
* Recompute import edges and symbol reference scores in-memory across all active graph nodes.
|
|
596
|
+
*/
|
|
597
|
+
_recomputeGraphEdgesAndSymbols() {
|
|
598
|
+
const nodes = Array.from(this.graphNodes.values()).sort((a, b) => comparePaths(a.path, b.path));
|
|
599
|
+
const edges = [];
|
|
600
|
+
|
|
601
|
+
if (!this.isMultiRepo || this.repos.length === 0) {
|
|
602
|
+
const root = this.projectRoot || process.cwd();
|
|
603
|
+
const resolver = new ImportResolver(root, nodes);
|
|
604
|
+
|
|
605
|
+
for (const node of nodes) {
|
|
606
|
+
const { edges: targets, unresolved } = resolver.resolve(node);
|
|
607
|
+
node.unresolvedImports = unresolved;
|
|
608
|
+
for (const { to, weight } of targets) {
|
|
609
|
+
edges.push({ from: node.path, to, weight });
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
} else {
|
|
613
|
+
// Multi-repo resolution: intra-repo ImportResolver per repo + cross-repo edge resolution
|
|
614
|
+
const nodesByAlias = new Map();
|
|
615
|
+
const resolversByAlias = new Map();
|
|
616
|
+
|
|
617
|
+
for (const repo of this.repos) {
|
|
618
|
+
nodesByAlias.set(repo.alias, []);
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
for (const node of nodes) {
|
|
622
|
+
const slashIdx = node.path.indexOf('/');
|
|
623
|
+
const alias = slashIdx > 0 ? node.path.slice(0, slashIdx) : null;
|
|
624
|
+
if (alias && nodesByAlias.has(alias)) {
|
|
625
|
+
const repoRelPath = node.path.slice(alias.length + 1);
|
|
626
|
+
nodesByAlias.get(alias).push({
|
|
627
|
+
origNode: node,
|
|
628
|
+
strippedNode: {
|
|
629
|
+
...node,
|
|
630
|
+
path: repoRelPath,
|
|
631
|
+
},
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
for (const repo of this.repos) {
|
|
637
|
+
const entries = nodesByAlias.get(repo.alias) || [];
|
|
638
|
+
const strippedNodes = entries.map((e) => e.strippedNode);
|
|
639
|
+
resolversByAlias.set(repo.alias, new ImportResolver(repo.root, strippedNodes));
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
for (const repo of this.repos) {
|
|
643
|
+
const entries = nodesByAlias.get(repo.alias) || [];
|
|
644
|
+
const resolver = resolversByAlias.get(repo.alias);
|
|
645
|
+
|
|
646
|
+
for (const { origNode, strippedNode } of entries) {
|
|
647
|
+
const { edges: intraTargets, unresolved: intraUnresolved } =
|
|
648
|
+
resolver.resolve(strippedNode);
|
|
649
|
+
const edgeWeights = new Map();
|
|
650
|
+
|
|
651
|
+
for (const { to, weight } of intraTargets) {
|
|
652
|
+
edgeWeights.set(`${repo.alias}/${to}`, weight);
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
// Attempt cross-repo resolution for imports
|
|
656
|
+
let crossResolvedCount = 0;
|
|
657
|
+
for (const imp of origNode.imports || []) {
|
|
658
|
+
if (!imp?.spec) continue;
|
|
659
|
+
const crossTarget = this._resolveCrossRepoSpec(
|
|
660
|
+
repo,
|
|
661
|
+
strippedNode.path,
|
|
662
|
+
imp.spec,
|
|
663
|
+
resolversByAlias
|
|
664
|
+
);
|
|
665
|
+
if (crossTarget && crossTarget !== origNode.path) {
|
|
666
|
+
edgeWeights.set(crossTarget, (edgeWeights.get(crossTarget) || 0) + 1);
|
|
667
|
+
crossResolvedCount++;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
origNode.unresolvedImports = Math.max(0, intraUnresolved - crossResolvedCount);
|
|
672
|
+
for (const [to, weight] of edgeWeights) {
|
|
673
|
+
edges.push({ from: origNode.path, to, weight });
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
edges.sort((a, b) => comparePaths(a.from, b.from) || comparePaths(a.to, b.to));
|
|
680
|
+
this.graphEdges = edges;
|
|
681
|
+
|
|
682
|
+
// Recompute symbolRefs across all nodes (intra-repo and cross-repo)
|
|
683
|
+
const definedNames = nodes.map(
|
|
684
|
+
(node) =>
|
|
685
|
+
new Set(
|
|
686
|
+
(node.methods || [])
|
|
687
|
+
.map((method) => method.name.split(' ').pop())
|
|
688
|
+
.filter((name) => name && name.length > 2)
|
|
689
|
+
)
|
|
690
|
+
);
|
|
691
|
+
const definers = new Map();
|
|
692
|
+
for (const names of definedNames) {
|
|
693
|
+
for (const name of names) {
|
|
694
|
+
definers.set(name, (definers.get(name) || 0) + 1);
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
const mentions = new Map();
|
|
698
|
+
for (const node of nodes) {
|
|
699
|
+
const ids = node.identifiers || new Set();
|
|
700
|
+
for (const id of ids) {
|
|
701
|
+
if (definers.has(id)) {
|
|
702
|
+
mentions.set(id, (mentions.get(id) || 0) + 1);
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
nodes.forEach((node, i) => {
|
|
707
|
+
let score = 0;
|
|
708
|
+
for (const name of definedNames[i]) {
|
|
709
|
+
const defining = definers.get(name);
|
|
710
|
+
score += Math.max(0, (mentions.get(name) || 0) - defining) / defining;
|
|
711
|
+
}
|
|
712
|
+
node.symbolRefs = score;
|
|
713
|
+
});
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
/**
|
|
717
|
+
* Check whether a relative path is denied by policy or ignored by scanner rules in its owning repo.
|
|
718
|
+
* @param {string} repoRelPath
|
|
719
|
+
* @param {string} fullPath
|
|
720
|
+
* @param {string} [repoRoot]
|
|
721
|
+
* @returns {{denied: boolean, ignored: boolean}}
|
|
722
|
+
*/
|
|
723
|
+
_checkPolicyAndIgnore(repoRelPath, fullPath, repoRoot = this.projectRoot) {
|
|
724
|
+
const targetRoot = repoRoot || this.projectRoot;
|
|
725
|
+
if (!targetRoot) return { denied: false, ignored: false };
|
|
726
|
+
try {
|
|
727
|
+
const scanner = new Scanner(targetRoot);
|
|
728
|
+
const denied = scanner.isDenied(repoRelPath);
|
|
729
|
+
const ignored = scanner.shouldIgnore(repoRelPath, fullPath);
|
|
730
|
+
return { denied, ignored };
|
|
731
|
+
} catch (_err) {
|
|
732
|
+
return { denied: false, ignored: false };
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
/**
|
|
737
|
+
* Synchronize a created or modified file with SqliteVectorStore and the knowledge graph.
|
|
738
|
+
* @param {Object} changeEvent
|
|
739
|
+
* @returns {Promise<{rag: Object|null, graph: Object|null}>}
|
|
740
|
+
*/
|
|
741
|
+
async syncFileChange(changeEvent) {
|
|
742
|
+
this._inferProjectRoot(changeEvent);
|
|
743
|
+
const ready = await this.ensureSyncReady();
|
|
744
|
+
if (!ready) return { rag: null, graph: null };
|
|
745
|
+
|
|
746
|
+
const { repo, repoRoot, repoRelPath, graphPath } = this._resolveChangeLocation(changeEvent);
|
|
747
|
+
const fullPath = changeEvent.path;
|
|
748
|
+
|
|
749
|
+
const { denied, ignored } = this._checkPolicyAndIgnore(repoRelPath, fullPath, repoRoot);
|
|
750
|
+
if (denied || ignored) {
|
|
751
|
+
// If previously indexed, purge from vector store and graph
|
|
752
|
+
const removedDocs = this.options.syncRag
|
|
753
|
+
? await this.vectorStore.removeDocument(graphPath)
|
|
754
|
+
: 0;
|
|
755
|
+
let removedGraph = { removedNode: 0, removedEdges: 0 };
|
|
756
|
+
if (this.options.syncGraph && this.graphNodes.has(graphPath)) {
|
|
757
|
+
this.graphNodes.delete(graphPath);
|
|
758
|
+
this._recomputeGraphEdgesAndSymbols();
|
|
759
|
+
removedGraph = await this.vectorStore.removeGraphNode(graphPath);
|
|
760
|
+
await this.vectorStore.syncGraph({
|
|
761
|
+
nodes: Array.from(this.graphNodes.values()),
|
|
762
|
+
edges: this.graphEdges,
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
const skippedPayload = {
|
|
766
|
+
file: graphPath,
|
|
767
|
+
reason: denied ? 'denied' : 'ignored',
|
|
768
|
+
removedDocs,
|
|
769
|
+
removedGraph,
|
|
770
|
+
};
|
|
771
|
+
this.emit('sync:skipped', skippedPayload);
|
|
772
|
+
return {
|
|
773
|
+
rag: { status: skippedPayload.reason, removed: removedDocs },
|
|
774
|
+
graph: { status: skippedPayload.reason, ...removedGraph },
|
|
775
|
+
};
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
let content;
|
|
779
|
+
try {
|
|
780
|
+
content = fs.readFileSync(fullPath, 'utf-8');
|
|
781
|
+
} catch {
|
|
782
|
+
return { rag: null, graph: null };
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
const sizeBytes = changeEvent.size ?? Buffer.byteLength(content, 'utf-8');
|
|
786
|
+
if (sizeBytes > this.options.maxFileBytes || content.includes('\0')) {
|
|
787
|
+
return {
|
|
788
|
+
rag: { status: 'skipped-size-or-binary', upserted: 0, skipped: 0 },
|
|
789
|
+
graph: { status: 'skipped-size-or-binary' },
|
|
790
|
+
};
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
const contentHash = crypto.createHash('sha256').update(content).digest('hex');
|
|
794
|
+
let ragResult = null;
|
|
795
|
+
let graphResult = null;
|
|
796
|
+
|
|
797
|
+
// 1. Incremental SQLite Vector Store upsert
|
|
798
|
+
if (this.options.syncRag && FileUtils.isText(fullPath)) {
|
|
799
|
+
let textToEmbed = content;
|
|
800
|
+
let redactionsCount = 0;
|
|
801
|
+
if (this.options.redactSecrets) {
|
|
802
|
+
const redacted = redactSecrets(content);
|
|
803
|
+
textToEmbed = redacted.text;
|
|
804
|
+
redactionsCount = redacted.findings.length;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
const { upserted, skipped } = await this.vectorStore.addDocuments([
|
|
808
|
+
{
|
|
809
|
+
id: graphPath,
|
|
810
|
+
text: textToEmbed,
|
|
811
|
+
metadata: {
|
|
812
|
+
source: 'watch-sync',
|
|
813
|
+
sourceId: graphPath,
|
|
814
|
+
path: fullPath,
|
|
815
|
+
relativePath: graphPath,
|
|
816
|
+
repo: repo?.alias || undefined,
|
|
817
|
+
extension: path.extname(fullPath),
|
|
818
|
+
filename: path.basename(fullPath),
|
|
819
|
+
size: sizeBytes,
|
|
820
|
+
updatedAt: new Date().toISOString(),
|
|
821
|
+
},
|
|
822
|
+
},
|
|
823
|
+
]);
|
|
824
|
+
|
|
825
|
+
ragResult = {
|
|
826
|
+
action: upserted > 0 ? 'upserted' : 'unchanged',
|
|
827
|
+
file: graphPath,
|
|
828
|
+
upserted,
|
|
829
|
+
skipped,
|
|
830
|
+
redactions: redactionsCount,
|
|
831
|
+
totalDocuments: this.vectorStore.count(),
|
|
832
|
+
};
|
|
833
|
+
this.emit('rag:synced', ragResult);
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
// 2. Incremental Knowledge Graph node & edge sync
|
|
837
|
+
if (this.options.syncGraph) {
|
|
838
|
+
const node = this._buildNodeFromContent(graphPath, fullPath, content, sizeBytes, contentHash);
|
|
839
|
+
if (node) {
|
|
840
|
+
this.graphNodes.set(graphPath, node);
|
|
841
|
+
this._recomputeGraphEdgesAndSymbols();
|
|
842
|
+
await this.vectorStore.syncGraph({
|
|
843
|
+
nodes: Array.from(this.graphNodes.values()),
|
|
844
|
+
edges: this.graphEdges,
|
|
845
|
+
});
|
|
846
|
+
|
|
847
|
+
const fileEdges = this.graphEdges.filter((e) => e.from === graphPath);
|
|
848
|
+
graphResult = {
|
|
849
|
+
action: 'upserted',
|
|
850
|
+
file: graphPath,
|
|
851
|
+
methods: node.methods.length,
|
|
852
|
+
outgoingEdges: fileEdges.length,
|
|
853
|
+
symbolRefs: node.symbolRefs,
|
|
854
|
+
totalNodes: this.graphNodes.size,
|
|
855
|
+
totalEdges: this.graphEdges.length,
|
|
856
|
+
};
|
|
857
|
+
this.emit('graph:updated', graphResult);
|
|
858
|
+
} else if (this.graphNodes.has(graphPath)) {
|
|
859
|
+
// File became minified or non-code; remove from graph
|
|
860
|
+
this.graphNodes.delete(graphPath);
|
|
861
|
+
this._recomputeGraphEdgesAndSymbols();
|
|
862
|
+
await this.vectorStore.removeGraphNode(graphPath);
|
|
863
|
+
await this.vectorStore.syncGraph({
|
|
864
|
+
nodes: Array.from(this.graphNodes.values()),
|
|
865
|
+
edges: this.graphEdges,
|
|
866
|
+
});
|
|
867
|
+
graphResult = {
|
|
868
|
+
action: 'removed',
|
|
869
|
+
file: graphPath,
|
|
870
|
+
totalNodes: this.graphNodes.size,
|
|
871
|
+
totalEdges: this.graphEdges.length,
|
|
872
|
+
};
|
|
873
|
+
this.emit('graph:updated', graphResult);
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
return { rag: ragResult, graph: graphResult };
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
/**
|
|
881
|
+
* Analyze changed file and update stats, vector embeddings, and knowledge graph.
|
|
35
882
|
* @param {FileChangeEvent} changeEvent
|
|
36
883
|
*/
|
|
37
884
|
async analyzeChange(changeEvent) {
|
|
38
885
|
logger.debug(`Analyzing change: ${changeEvent.relativePath}`);
|
|
39
886
|
|
|
40
887
|
const startTime = Date.now();
|
|
888
|
+
this._inferProjectRoot(changeEvent);
|
|
41
889
|
|
|
42
890
|
try {
|
|
43
|
-
// If file deleted, remove from cache
|
|
891
|
+
// If file deleted, remove from cache, vector store, and knowledge graph
|
|
44
892
|
if (!changeEvent.exists) {
|
|
45
|
-
this.handleFileDeleted(changeEvent);
|
|
893
|
+
await this.handleFileDeleted(changeEvent);
|
|
894
|
+
return;
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
const { repoRoot, repoRelPath, graphPath } = this._resolveChangeLocation(changeEvent);
|
|
898
|
+
const { denied, ignored } = this._checkPolicyAndIgnore(
|
|
899
|
+
repoRelPath,
|
|
900
|
+
changeEvent.path,
|
|
901
|
+
repoRoot
|
|
902
|
+
);
|
|
903
|
+
if (denied || ignored) {
|
|
904
|
+
if (this.options.syncRag || this.options.syncGraph) {
|
|
905
|
+
await this.syncFileChange(changeEvent);
|
|
906
|
+
}
|
|
46
907
|
return;
|
|
47
908
|
}
|
|
48
909
|
|
|
49
910
|
// Analyze file
|
|
50
911
|
const fileInfo = {
|
|
51
912
|
path: changeEvent.path,
|
|
52
|
-
relativePath: changeEvent.relativePath,
|
|
913
|
+
relativePath: this.isMultiRepo ? graphPath : changeEvent.relativePath,
|
|
53
914
|
name: path.basename(changeEvent.path),
|
|
54
915
|
extension: path.extname(changeEvent.path),
|
|
55
916
|
size: changeEvent.size,
|
|
@@ -66,11 +927,18 @@ export class IncrementalAnalyzer extends EventEmitter {
|
|
|
66
927
|
// Update aggregate stats
|
|
67
928
|
this.updateAggregateStats(oldAnalysis, analysis);
|
|
68
929
|
|
|
930
|
+
// Synchronize SQLite vector store & knowledge graph if enabled
|
|
931
|
+
let sync = null;
|
|
932
|
+
if (this.options.syncRag || this.options.syncGraph) {
|
|
933
|
+
sync = await this.syncFileChange(changeEvent);
|
|
934
|
+
}
|
|
935
|
+
|
|
69
936
|
const elapsed = Date.now() - startTime;
|
|
70
937
|
|
|
71
938
|
this.emit('analysis:complete', {
|
|
72
|
-
file: changeEvent.relativePath,
|
|
939
|
+
file: this.isMultiRepo ? graphPath : changeEvent.relativePath,
|
|
73
940
|
analysis,
|
|
941
|
+
sync,
|
|
74
942
|
elapsed,
|
|
75
943
|
cached: false,
|
|
76
944
|
});
|
|
@@ -87,11 +955,12 @@ export class IncrementalAnalyzer extends EventEmitter {
|
|
|
87
955
|
}
|
|
88
956
|
|
|
89
957
|
/**
|
|
90
|
-
* Handle file deletion
|
|
958
|
+
* Handle file deletion across analysis cache, SqliteVectorStore, and knowledge graph.
|
|
91
959
|
* @param {FileChangeEvent} changeEvent
|
|
92
960
|
*/
|
|
93
|
-
handleFileDeleted(changeEvent) {
|
|
961
|
+
async handleFileDeleted(changeEvent) {
|
|
94
962
|
const oldAnalysis = this.fileCache.get(changeEvent.path);
|
|
963
|
+
const { graphPath } = this._resolveChangeLocation(changeEvent);
|
|
95
964
|
|
|
96
965
|
if (oldAnalysis) {
|
|
97
966
|
// Remove from cache
|
|
@@ -111,13 +980,62 @@ export class IncrementalAnalyzer extends EventEmitter {
|
|
|
111
980
|
this.aggregateStats.byLanguage[lang].size -= oldAnalysis.size;
|
|
112
981
|
}
|
|
113
982
|
}
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
let sync = null;
|
|
986
|
+
if (this.options.syncRag || this.options.syncGraph) {
|
|
987
|
+
const ready = await this.ensureSyncReady();
|
|
988
|
+
if (ready) {
|
|
989
|
+
let ragResult = null;
|
|
990
|
+
let graphResult = null;
|
|
991
|
+
|
|
992
|
+
if (this.options.syncRag) {
|
|
993
|
+
const removedByRel = await this.vectorStore.removeDocument(graphPath);
|
|
994
|
+
const removedByAbs =
|
|
995
|
+
changeEvent.path && changeEvent.path !== graphPath
|
|
996
|
+
? await this.vectorStore.removeDocument(changeEvent.path)
|
|
997
|
+
: 0;
|
|
998
|
+
const removed = removedByRel + removedByAbs;
|
|
999
|
+
ragResult = {
|
|
1000
|
+
action: 'deleted',
|
|
1001
|
+
file: graphPath,
|
|
1002
|
+
removed,
|
|
1003
|
+
totalDocuments: this.vectorStore.count(),
|
|
1004
|
+
};
|
|
1005
|
+
this.emit('rag:synced', ragResult);
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
if (this.options.syncGraph) {
|
|
1009
|
+
this.graphNodes.delete(graphPath);
|
|
1010
|
+
this._recomputeGraphEdgesAndSymbols();
|
|
1011
|
+
const { removedNode, removedEdges } = await this.vectorStore.removeGraphNode(graphPath);
|
|
1012
|
+
await this.vectorStore.syncGraph({
|
|
1013
|
+
nodes: Array.from(this.graphNodes.values()),
|
|
1014
|
+
edges: this.graphEdges,
|
|
1015
|
+
});
|
|
1016
|
+
graphResult = {
|
|
1017
|
+
action: 'deleted',
|
|
1018
|
+
file: graphPath,
|
|
1019
|
+
removedNode,
|
|
1020
|
+
removedEdges,
|
|
1021
|
+
totalNodes: this.graphNodes.size,
|
|
1022
|
+
totalEdges: this.graphEdges.length,
|
|
1023
|
+
};
|
|
1024
|
+
this.emit('graph:updated', graphResult);
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
sync = { rag: ragResult, graph: graphResult };
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
114
1030
|
|
|
1031
|
+
if (oldAnalysis || sync) {
|
|
115
1032
|
this.emit('file:deleted', {
|
|
116
|
-
file: changeEvent.relativePath,
|
|
117
|
-
oldTokens: oldAnalysis.tokens,
|
|
1033
|
+
file: this.isMultiRepo ? graphPath : changeEvent.relativePath,
|
|
1034
|
+
oldTokens: oldAnalysis ? oldAnalysis.tokens : 0,
|
|
1035
|
+
sync,
|
|
118
1036
|
});
|
|
119
1037
|
|
|
120
|
-
logger.debug(`File deleted from cache: ${
|
|
1038
|
+
logger.debug(`File deleted from cache/store: ${graphPath}`);
|
|
121
1039
|
}
|
|
122
1040
|
}
|
|
123
1041
|
|
|
@@ -166,6 +1084,23 @@ export class IncrementalAnalyzer extends EventEmitter {
|
|
|
166
1084
|
this.emit('stats:updated', this.aggregateStats);
|
|
167
1085
|
}
|
|
168
1086
|
|
|
1087
|
+
/**
|
|
1088
|
+
* Get current knowledge graph snapshot (from SQLite store if initialized, else in-memory).
|
|
1089
|
+
* @returns {Promise<{nodes: Array<Object>, edges: Array<{from: string, to: string, weight: number}>}>}
|
|
1090
|
+
*/
|
|
1091
|
+
async getGraph() {
|
|
1092
|
+
if (this.vectorStore) {
|
|
1093
|
+
return this.vectorStore.getGraph();
|
|
1094
|
+
}
|
|
1095
|
+
const nodes = Array.from(this.graphNodes.values())
|
|
1096
|
+
.map(({ identifiers: _identifiers, ...rest }) => rest)
|
|
1097
|
+
.sort((a, b) => comparePaths(a.path, b.path));
|
|
1098
|
+
return {
|
|
1099
|
+
nodes,
|
|
1100
|
+
edges: [...this.graphEdges],
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
|
|
169
1104
|
/**
|
|
170
1105
|
* Get current aggregate statistics
|
|
171
1106
|
* @returns {object}
|
|
@@ -191,6 +1126,15 @@ export class IncrementalAnalyzer extends EventEmitter {
|
|
|
191
1126
|
logger.info('Analysis cache cleared');
|
|
192
1127
|
}
|
|
193
1128
|
|
|
1129
|
+
/**
|
|
1130
|
+
* Close underlying vector store connection if open.
|
|
1131
|
+
*/
|
|
1132
|
+
close() {
|
|
1133
|
+
if (this.vectorStore && typeof this.vectorStore.close === 'function') {
|
|
1134
|
+
this.vectorStore.close();
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
|
|
194
1138
|
/**
|
|
195
1139
|
* Get cache statistics
|
|
196
1140
|
* @returns {object}
|
|
@@ -199,6 +1143,10 @@ export class IncrementalAnalyzer extends EventEmitter {
|
|
|
199
1143
|
return {
|
|
200
1144
|
cachedFiles: this.fileCache.size,
|
|
201
1145
|
cacheSize: this.fileCache.size * 1024, // Rough estimate
|
|
1146
|
+
graphNodes: this.graphNodes.size,
|
|
1147
|
+
graphEdges: this.graphEdges.length,
|
|
1148
|
+
vectorDocuments: this.vectorStore ? this.vectorStore.count() : 0,
|
|
1149
|
+
repos: this.repos.map((r) => ({ root: r.root, alias: r.alias })),
|
|
202
1150
|
};
|
|
203
1151
|
}
|
|
204
1152
|
}
|