graph-indexer 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +412 -0
- package/README.md +228 -350
- package/SECURITY.md +68 -0
- package/TRADEMARK.md +31 -0
- package/cli-ui.mjs +78 -0
- package/config.mjs +269 -0
- package/daemon-ctl.mjs +175 -0
- package/daemon-lock.mjs +69 -0
- package/embedders/python/mlx_embed_server.py +61 -0
- package/embedders/python/requirements-mlx.txt +5 -0
- package/embedders/setup-mlx.mjs +110 -0
- package/embeddings.mjs +489 -0
- package/engine/binary.mjs +399 -0
- package/{core-engine.mjs → engine/memory.mjs} +359 -249
- package/engine/sqlite.mjs +863 -0
- package/enrichment.mjs +426 -0
- package/git-signals.mjs +248 -0
- package/indexer.mjs +216 -77
- package/init.mjs +1604 -170
- package/layout.mjs +123 -0
- package/mcp/format.mjs +61 -0
- package/mcp/tools.mjs +1157 -0
- package/mcp/topology.mjs +551 -0
- package/mcp-server.mjs +118 -573
- package/package.json +56 -16
- package/parse/extractor.mjs +617 -0
- package/parse/imports.mjs +365 -0
- package/parse/languages.mjs +89 -0
- package/parse/metadata.mjs +655 -0
- package/parse/routes.mjs +327 -0
- package/prompts/CORE.md +86 -0
- package/prompts/DOMAIN_TEMPLATE.md +46 -0
- package/prompts/INTEGRATION.md +123 -0
- package/prompts/frameworks/ANDROID.md +17 -0
- package/prompts/frameworks/ASPNET_CORE.md +19 -0
- package/prompts/frameworks/FASTAPI_DJANGO.md +19 -0
- package/prompts/frameworks/LARAVEL_SYMFONY.md +22 -0
- package/prompts/frameworks/NODE_EXPRESS_NESTJS.md +18 -0
- package/prompts/frameworks/RAILS.md +20 -0
- package/prompts/frameworks/REACT.md +20 -0
- package/prompts/frameworks/SPRING_BOOT.md +19 -0
- package/prompts/languages/BASH.md +22 -0
- package/prompts/languages/C.md +21 -0
- package/prompts/languages/CSHARP.md +24 -0
- package/prompts/languages/CSS_SCSS.md +22 -0
- package/prompts/languages/GO.md +23 -0
- package/prompts/languages/JAVA.md +23 -0
- package/prompts/languages/JAVASCRIPT_TYPESCRIPT.md +24 -0
- package/prompts/languages/KOTLIN.md +24 -0
- package/prompts/languages/PHP.md +23 -0
- package/prompts/languages/PYTHON.md +23 -0
- package/prompts/languages/RUBY.md +23 -0
- package/prompts/languages/RUST.md +21 -0
- package/prompts/languages/SWIFT.md +21 -0
- package/search-core.mjs +627 -0
- package/storage.mjs +68 -0
- package/watch-daemon.mjs +204 -69
- package/PROMPT.md +0 -143
- package/parser-utils.mjs +0 -861
package/storage.mjs
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file storage.mjs
|
|
3
|
+
* @description Storage backend factory. Selects the in-memory engine (default,
|
|
4
|
+
* zero-dependency) or the disk-backed SQLite store based on config,
|
|
5
|
+
* and documents the single read contract that the MCP tools depend
|
|
6
|
+
* on so they remain backend-agnostic.
|
|
7
|
+
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
8
|
+
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
9
|
+
* @license MIT
|
|
10
|
+
*
|
|
11
|
+
* ── Store contract (implemented by both MemoryGraphIndex and SqliteGraphStore) ──
|
|
12
|
+
* load() Prepare for queries (open db / parse json).
|
|
13
|
+
* get backend() 'memory' | 'sqlite'.
|
|
14
|
+
* get graph() { dependencies, importedBy } (file-level).
|
|
15
|
+
* searchHybrid(q, vec, topK, minScore, exactBoost) → [{ score, chunk }]
|
|
16
|
+
* getChunk(id) → chunk | null
|
|
17
|
+
* getChunksByFile(path) → chunk[]
|
|
18
|
+
* resolveSymbol(name) → chunk[] (exact, case-insensitive)
|
|
19
|
+
* findCallers(funcName) → chunk[]
|
|
20
|
+
* findReferers(symbol) → chunk[] (type_refs / extends matches)
|
|
21
|
+
* findRoutes({method, pathPrefix}) → route[] (HTTP route → handler chunk)
|
|
22
|
+
* iterateChunks() → Iterable<chunk> (cursor on SQLite)
|
|
23
|
+
* getDependencies(path) / getImportedBy(path) → string[]
|
|
24
|
+
* chunkCount() / fileCount() / symbolCount() / vectorCount() → number
|
|
25
|
+
* stats() → engine health facts
|
|
26
|
+
* close() Release fds / db handles.
|
|
27
|
+
*/
|
|
28
|
+
import fs from 'fs';
|
|
29
|
+
import { MemoryGraphIndex } from './engine/memory.mjs';
|
|
30
|
+
|
|
31
|
+
// 'auto' storage keeps the index in memory until a repo is large enough that the
|
|
32
|
+
// disk-backed SQLite store earns its slightly higher per-query latency. The indexer
|
|
33
|
+
// decides by the real chunk count it just built; readers decide by which artifact
|
|
34
|
+
// exists on disk (see resolveBackend).
|
|
35
|
+
export const AUTO_SQLITE_CHUNK_THRESHOLD = 15000;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Resolve the abstract storage setting ('auto'|'memory'|'sqlite') to a concrete
|
|
39
|
+
* read backend. Explicit settings win; 'auto' picks SQLite when a SQLite artifact
|
|
40
|
+
* is present (the indexer writes one only for large repos and removes the other
|
|
41
|
+
* backend's artifact, so presence is unambiguous), else the in-memory JSON index.
|
|
42
|
+
*
|
|
43
|
+
* @param {object} config Resolved config from config.mjs.
|
|
44
|
+
* @returns {'memory'|'sqlite'}
|
|
45
|
+
*/
|
|
46
|
+
export function resolveBackend(config) {
|
|
47
|
+
if (config.storage === 'sqlite') return 'sqlite';
|
|
48
|
+
if (config.storage === 'memory') return 'memory';
|
|
49
|
+
try { if (fs.existsSync(config.sqlitePath)) return 'sqlite'; } catch { /* fall through to memory */ }
|
|
50
|
+
return 'memory';
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Construct (but do not yet load) the configured store.
|
|
55
|
+
*
|
|
56
|
+
* @param {object} config Resolved config from config.mjs.
|
|
57
|
+
* @param {object} [opts]
|
|
58
|
+
* @param {boolean} [opts.cacheEmbeddings] Eager vector cache (in-memory backend only).
|
|
59
|
+
* @returns {Promise<object>} a store implementing the contract above.
|
|
60
|
+
*/
|
|
61
|
+
export async function createStore(config, { cacheEmbeddings = false } = {}) {
|
|
62
|
+
if (resolveBackend(config) === 'sqlite') {
|
|
63
|
+
// Imported lazily so the default path never loads node:sqlite.
|
|
64
|
+
const { SqliteGraphStore } = await import('./engine/sqlite.mjs');
|
|
65
|
+
return new SqliteGraphStore(config.sqlitePath, { embeddingPath: config.embeddingPath });
|
|
66
|
+
}
|
|
67
|
+
return new MemoryGraphIndex(config.indexPath, { cacheEmbeddings });
|
|
68
|
+
}
|
package/watch-daemon.mjs
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
3
|
* @file watch-daemon.mjs
|
|
4
|
-
* @description Native FileSystem Watcher Daemon
|
|
4
|
+
* @description Native FileSystem Watcher Daemon that keeps the configured index
|
|
5
|
+
* backend fresh incrementally. Backend-agnostic: file changes are
|
|
6
|
+
* parsed once (Tree-sitter), optionally enriched (LLM cache-first)
|
|
7
|
+
* and embedded (Ollama), then handed to the store's applyFileUpdate
|
|
8
|
+
* — the in-memory engine persists a debounced JSON snapshot, the
|
|
9
|
+
* SQLite store commits a per-file WAL transaction that running MCP
|
|
10
|
+
* servers pick up live via PRAGMA data_version. No more "SQLite
|
|
11
|
+
* requires a full re-index on every change".
|
|
5
12
|
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
6
13
|
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
7
14
|
* @license MIT
|
|
@@ -19,36 +26,158 @@
|
|
|
19
26
|
import fs from 'fs';
|
|
20
27
|
import path from 'path';
|
|
21
28
|
import chokidar from 'chokidar';
|
|
22
|
-
import {
|
|
29
|
+
import { resolveConfig } from './config.mjs';
|
|
30
|
+
import { ensureDataDir, migrateLegacyLayout } from './layout.mjs';
|
|
31
|
+
import { acquireLock, releaseLock, readPid } from './daemon-lock.mjs';
|
|
32
|
+
import { createStore, resolveBackend } from './storage.mjs';
|
|
23
33
|
import {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
} from './
|
|
34
|
+
embeddingKeyFor, computePageRank, TEST_FILE_RE, EXAMPLE_DIR_RE,
|
|
35
|
+
summaryEmbeddingText, SUMMARY_VEC_SUFFIX, WINDOW_VEC_SUFFIX, embeddingWindows,
|
|
36
|
+
} from './search-core.mjs';
|
|
37
|
+
import { MAX_FILE_SIZE_BYTES, buildIgnoreFilter, extractImportsFromAST, extractSemanticChunks } from './parse/extractor.mjs';
|
|
38
|
+
import { getParserForFile } from './parse/languages.mjs';
|
|
39
|
+
import { resolveLocalImports, buildEmbeddingPayload, fullBodyForEmbedding } from './parse/imports.mjs';
|
|
40
|
+
import { createEmbedder, readEmbedMeta } from './embeddings.mjs';
|
|
41
|
+
import {
|
|
42
|
+
loadEnrichmentCache, saveEnrichmentCache, attachEnrichment,
|
|
43
|
+
ollamaGenerate, parseEnrichResponse, buildEnrichPrompt, resolveEnrichModel,
|
|
44
|
+
} from './enrichment.mjs';
|
|
45
|
+
|
|
46
|
+
const config = resolveConfig();
|
|
47
|
+
const PROJECT_ROOT = config.projectRoot;
|
|
28
48
|
|
|
29
|
-
|
|
30
|
-
|
|
49
|
+
// Embed incremental updates with the SAME provider/model the index was built with
|
|
50
|
+
// (stamped in the embeddings-bin sidecar), so live vectors match the bootstrap
|
|
51
|
+
// ones. Built lazily and cached for the daemon's lifetime.
|
|
52
|
+
let _embedder = null;
|
|
53
|
+
async function getEmbedder() {
|
|
54
|
+
if (_embedder) return _embedder;
|
|
55
|
+
const m = readEmbedMeta(config.embeddingPath);
|
|
56
|
+
_embedder = await createEmbedder(config, m?.provider
|
|
57
|
+
? { provider: m.provider, model: m.model }
|
|
58
|
+
: { provider: config.embeddingsEnabled ? 'ollama' : 'off', model: config.embedModel });
|
|
59
|
+
return _embedder;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// ─── Single-instance guard ──────────────────────────────────────────────────────
|
|
63
|
+
// Exactly one daemon may run per project. Acquire the PID lock BEFORE doing any
|
|
64
|
+
// work (loading the store, opening the db) so a redundant launch — via the MCP
|
|
65
|
+
// server, `idx-daemon start`, or a stray `idx-watch` — exits immediately instead
|
|
66
|
+
// of racing a second watcher onto the same index.
|
|
67
|
+
ensureDataDir(PROJECT_ROOT);
|
|
68
|
+
migrateLegacyLayout(PROJECT_ROOT);
|
|
69
|
+
if (!acquireLock(config.pidFile)) {
|
|
70
|
+
process.stderr.write(
|
|
71
|
+
`✋ A graph-indexer daemon is already running for this project `
|
|
72
|
+
+ `(PID ${readPid(config.pidFile)}). Exiting — only one runs per project.\n`
|
|
73
|
+
);
|
|
74
|
+
process.exit(0);
|
|
75
|
+
}
|
|
31
76
|
|
|
32
77
|
const ignoreFilter = buildIgnoreFilter(PROJECT_ROOT);
|
|
33
|
-
const db =
|
|
78
|
+
const db = await createStore(config, { cacheEmbeddings: true });
|
|
34
79
|
db.load();
|
|
35
80
|
|
|
81
|
+
// ─── Enrichment (cache-first, live for core files) ──────────────────────────────
|
|
82
|
+
// Cached entries re-attach for free on every change. When enrichment is enabled
|
|
83
|
+
// and the changed file sits in the PageRank core, new/changed chunks are enriched
|
|
84
|
+
// live (best-effort, low concurrency) so edits don't silently lose their semantic
|
|
85
|
+
// metadata until the next full index run.
|
|
86
|
+
const enrichCache = config.enrichment.enabled
|
|
87
|
+
? loadEnrichmentCache(config.enrichmentCachePath)
|
|
88
|
+
: null;
|
|
89
|
+
let enrichCacheDirty = false;
|
|
90
|
+
let saveCacheTimer = null;
|
|
91
|
+
function scheduleEnrichCacheSave() {
|
|
92
|
+
if (!enrichCacheDirty) return;
|
|
93
|
+
if (saveCacheTimer) clearTimeout(saveCacheTimer);
|
|
94
|
+
saveCacheTimer = setTimeout(() => {
|
|
95
|
+
saveCacheTimer = null;
|
|
96
|
+
try { saveEnrichmentCache(config.enrichmentCachePath, enrichCache); enrichCacheDirty = false; }
|
|
97
|
+
catch (err) { process.stderr.write(`[daemon] ⚠️ enrichment cache save failed: ${err.message}\n`); }
|
|
98
|
+
}, 3000);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// "auto" resolves to the strongest available local code model, same policy as the bootstrap indexer.
|
|
102
|
+
let _enrichModel = null;
|
|
103
|
+
async function getEnrichModel() {
|
|
104
|
+
if (_enrichModel) return _enrichModel;
|
|
105
|
+
_enrichModel = await resolveEnrichModel(config.enrichment.model, config.ollamaHost);
|
|
106
|
+
return _enrichModel;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
let _coreFiles = null;
|
|
110
|
+
function isEnrichableFile(filename) {
|
|
111
|
+
if (TEST_FILE_RE.test(filename) || EXAMPLE_DIR_RE.test(filename)) return false;
|
|
112
|
+
if (config.enrichment.coreRatio >= 1) return true; // all production files
|
|
113
|
+
if (!_coreFiles) {
|
|
114
|
+
const pr = computePageRank(db.graph);
|
|
115
|
+
const files = Array.from(pr.keys()).sort((a, b) => (pr.get(b) ?? 0) - (pr.get(a) ?? 0));
|
|
116
|
+
const coreCount = Math.max(1, Math.ceil(files.length * config.enrichment.coreRatio));
|
|
117
|
+
_coreFiles = new Set(files.slice(0, coreCount));
|
|
118
|
+
}
|
|
119
|
+
return _coreFiles.has(filename);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function enrichChunks(filename, chunks) {
|
|
123
|
+
if (!enrichCache) return;
|
|
124
|
+
const pending = [];
|
|
125
|
+
for (const chunk of chunks) {
|
|
126
|
+
if (!chunk.content_hash) continue;
|
|
127
|
+
if (attachEnrichment(chunk, enrichCache.get(chunk.content_hash))) continue;
|
|
128
|
+
pending.push(chunk);
|
|
129
|
+
}
|
|
130
|
+
if (pending.length === 0 || !isEnrichableFile(filename)) return;
|
|
131
|
+
// Live enrichment of changed core chunks: sequential and best-effort — a file
|
|
132
|
+
// save touches a handful of chunks, so latency stays low and failures only
|
|
133
|
+
// mean "no enrichment until the next full index run".
|
|
134
|
+
const model = await getEnrichModel();
|
|
135
|
+
for (const chunk of pending) {
|
|
136
|
+
if ((chunk.end_line - chunk.start_line) < 4) continue; // trivial stubs
|
|
137
|
+
const raw = await ollamaGenerate(buildEnrichPrompt(chunk), {
|
|
138
|
+
model, ollamaHost: config.ollamaHost, timeoutMs: 20000,
|
|
139
|
+
});
|
|
140
|
+
const parsed = parseEnrichResponse(raw);
|
|
141
|
+
if (!parsed) break; // model unreachable — stop trying for this batch
|
|
142
|
+
chunk.summary = parsed.summary;
|
|
143
|
+
chunk.concepts = parsed.concepts;
|
|
144
|
+
chunk.hyde = parsed.hyde;
|
|
145
|
+
enrichCache.set(chunk.content_hash, {
|
|
146
|
+
summary: parsed.summary, concepts: parsed.concepts, model,
|
|
147
|
+
});
|
|
148
|
+
enrichCacheDirty = true;
|
|
149
|
+
}
|
|
150
|
+
scheduleEnrichCacheSave();
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// ─── Incremental file sync ──────────────────────────────────────────────────────
|
|
154
|
+
|
|
155
|
+
// Catch-up scan: chokidar fires 'add' for every existing file at startup. Files
|
|
156
|
+
// not modified since the index artifact was last written are already indexed —
|
|
157
|
+
// skipping them turns daemon start from O(repo) parse work into O(changed files),
|
|
158
|
+
// while still picking up edits made while the daemon was down.
|
|
159
|
+
const BACKEND = resolveBackend(config); // 'auto' → concrete backend by artifact presence.
|
|
160
|
+
const ARTIFACT_PATH = BACKEND === 'sqlite' ? config.sqlitePath : config.indexPath;
|
|
161
|
+
let indexBuiltAt = 0;
|
|
162
|
+
try { indexBuiltAt = fs.statSync(ARTIFACT_PATH).mtimeMs; } catch { /* no index yet → full scan */ }
|
|
163
|
+
let initialScan = true;
|
|
164
|
+
|
|
36
165
|
async function processFileChange(absolutePath) {
|
|
37
166
|
const filename = path.relative(PROJECT_ROOT, absolutePath).replace(/\\/g, '/');
|
|
38
167
|
if (ignoreFilter.ignores(filename)) return;
|
|
39
168
|
if (filename.includes('.bundle.') || filename.includes('.min.')) return;
|
|
169
|
+
if (filename.startsWith('code-index.')) return; // never index our own artifacts
|
|
40
170
|
|
|
41
171
|
try {
|
|
172
|
+
if (initialScan && indexBuiltAt > 0 && fs.existsSync(absolutePath)
|
|
173
|
+
&& fs.statSync(absolutePath).mtimeMs <= indexBuiltAt) {
|
|
174
|
+
return; // unchanged since the last index write
|
|
175
|
+
}
|
|
176
|
+
|
|
42
177
|
if (!fs.existsSync(absolutePath)) {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
db.chunks.delete(id);
|
|
47
|
-
db.removeVector(id);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
db.updateFileGraph(filename, []);
|
|
51
|
-
db.saveDebounced();
|
|
178
|
+
if (typeof db.removeFile === 'function') db.removeFile(filename);
|
|
179
|
+
else db.applyFileUpdate(filename, { chunks: [], imports: [] });
|
|
180
|
+
_coreFiles = null;
|
|
52
181
|
process.stderr.write(`[daemon] 🗑️ Purged: ${filename}\n`);
|
|
53
182
|
return;
|
|
54
183
|
}
|
|
@@ -56,7 +185,7 @@ async function processFileChange(absolutePath) {
|
|
|
56
185
|
const stats = fs.statSync(absolutePath);
|
|
57
186
|
if (stats.size > MAX_FILE_SIZE_BYTES) return;
|
|
58
187
|
|
|
59
|
-
|
|
188
|
+
const content = fs.readFileSync(absolutePath, 'utf-8');
|
|
60
189
|
if (!content.trim()) return;
|
|
61
190
|
|
|
62
191
|
const ext = path.extname(absolutePath);
|
|
@@ -65,67 +194,57 @@ async function processFileChange(absolutePath) {
|
|
|
65
194
|
|
|
66
195
|
const tree = parser.parse((offset) => offset < content.length ? content.slice(offset, offset + 4096) : null);
|
|
67
196
|
const imports = resolveLocalImports(extractImportsFromAST(tree.rootNode, ext), filename, PROJECT_ROOT);
|
|
68
|
-
db.updateFileGraph(filename, imports);
|
|
69
|
-
|
|
70
197
|
const newChunks = extractSemanticChunks(tree.rootNode, filename, content, ext);
|
|
198
|
+
_coreFiles = null; // dependency graph may change → PageRank core set is stale
|
|
71
199
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
db.chunks.delete(id);
|
|
76
|
-
db.removeVector(id); // invalidates flat matrix so next search rebuilds it
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
const chunksToEmbed = newChunks.filter(c => !db.embeddingCache.has(c.content_hash));
|
|
200
|
+
// Enrichment BEFORE embedding: summary rides the vector payload and
|
|
201
|
+
// embeddingKeyFor accounts for it, so cache lookups stay correct.
|
|
202
|
+
await enrichChunks(filename, newChunks);
|
|
81
203
|
|
|
204
|
+
// Identical embedding payloads to indexer.mjs (shared helpers) so incremental
|
|
205
|
+
// updates match the bootstrap index, including summary-only vectors for enriched chunks.
|
|
206
|
+
const chunksToEmbed = newChunks.filter(c => !db.hasEmbedding(embeddingKeyFor(c)));
|
|
207
|
+
const embeddings = new Map();
|
|
82
208
|
if (chunksToEmbed.length > 0) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
}
|
|
95
|
-
});
|
|
209
|
+
const entries = [];
|
|
210
|
+
for (const c of chunksToEmbed) {
|
|
211
|
+
const baseKey = embeddingKeyFor(c);
|
|
212
|
+
// Oversized definitions embed their full body (windowed) while the lexical path
|
|
213
|
+
// keeps the 3000-char snippet — parity with how indexer.mjs builds payloads.
|
|
214
|
+
const payload = buildEmbeddingPayload(c, imports, fullBodyForEmbedding(c, content));
|
|
215
|
+
entries.push({ key: baseKey, text: payload });
|
|
216
|
+
const windows = embeddingWindows(payload); // tail windows for oversized chunks
|
|
217
|
+
for (let i = 1; i < windows.length; i++) entries.push({ key: baseKey + WINDOW_VEC_SUFFIX + i, text: windows[i] });
|
|
218
|
+
const sText = summaryEmbeddingText(c);
|
|
219
|
+
if (sText) entries.push({ key: baseKey + SUMMARY_VEC_SUFFIX, text: sText });
|
|
96
220
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
221
|
+
const embedder = await getEmbedder();
|
|
222
|
+
const embeddingsMatrix = await embedder.embedDocuments(entries.map(e => e.text));
|
|
223
|
+
if (embeddingsMatrix && embeddingsMatrix.length === entries.length) {
|
|
224
|
+
entries.forEach((entry, j) => {
|
|
225
|
+
if (embeddingsMatrix[j]) embeddings.set(entry.key, new Float32Array(embeddingsMatrix[j]));
|
|
226
|
+
});
|
|
102
227
|
}
|
|
103
|
-
|
|
104
|
-
// Unified lexical re-indexing of neighborhood
|
|
105
|
-
const cleanDeps = imports.map(d => d.split('/').pop().split('.')[0]);
|
|
106
|
-
const enrichedContext = [
|
|
107
|
-
chunk.name,
|
|
108
|
-
chunk.docstring || '',
|
|
109
|
-
cleanDeps.join(' '),
|
|
110
|
-
(chunk.calls || []).join(' '),
|
|
111
|
-
(chunk.params || []).join(' '),
|
|
112
|
-
chunk.return_type || '',
|
|
113
|
-
chunk.class_context ? `${chunk.class_context}.${chunk.name}` : '',
|
|
114
|
-
chunk.code_snippet
|
|
115
|
-
].join(' ');
|
|
116
|
-
|
|
117
|
-
db._indexLexical(chunk.id, enrichedContext, chunk.file_path);
|
|
118
|
-
db.chunks.set(chunk.id, chunk);
|
|
119
228
|
}
|
|
120
229
|
|
|
121
|
-
db.
|
|
122
|
-
process.stderr.write(
|
|
230
|
+
db.applyFileUpdate(filename, { chunks: newChunks, imports, embeddings });
|
|
231
|
+
process.stderr.write(
|
|
232
|
+
`[daemon] 🔄 Synced: ${filename} (${newChunks.length} chunks, `
|
|
233
|
+
+ `${newChunks.length - chunksToEmbed.length} cached vectors, ${BACKEND})\n`
|
|
234
|
+
);
|
|
123
235
|
} catch (err) {
|
|
124
236
|
process.stderr.write(`[daemon] ❌ Error in ${filename}: ${err.message}\n`);
|
|
125
237
|
}
|
|
126
238
|
}
|
|
127
239
|
|
|
128
|
-
|
|
240
|
+
// Serialize updates: chokidar can fire bursts (git checkout, format-on-save across
|
|
241
|
+
// files); per-file work must not interleave inside the store.
|
|
242
|
+
let queue = Promise.resolve();
|
|
243
|
+
function enqueue(absolutePath) {
|
|
244
|
+
queue = queue.then(() => processFileChange(absolutePath)).catch(() => {});
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
process.stderr.write(`🚀 Watcher Daemon started in: ${PROJECT_ROOT} (backend: ${BACKEND})\n`);
|
|
129
248
|
|
|
130
249
|
// Only watch source files we can actually index. Critically, this prevents
|
|
131
250
|
// chokidar from descending into node_modules / dist / .git / .gitignored dirs —
|
|
@@ -136,6 +255,7 @@ function shouldIgnore(absPath) {
|
|
|
136
255
|
const rel = path.relative(PROJECT_ROOT, absPath).replace(/\\/g, '/');
|
|
137
256
|
if (!rel) return false; // the project root itself
|
|
138
257
|
if (path.basename(absPath).startsWith('.')) return true; // dotfiles / dot-dirs
|
|
258
|
+
if (rel.startsWith('code-index.')) return true; // our own artifacts (.db/-wal/.bin/.json)
|
|
139
259
|
try { return ignoreFilter.ignores(rel); } // node_modules, dist, .gitignore, …
|
|
140
260
|
catch { return false; }
|
|
141
261
|
}
|
|
@@ -145,5 +265,20 @@ const watcher = chokidar.watch(PROJECT_ROOT, {
|
|
|
145
265
|
persistent: true,
|
|
146
266
|
awaitWriteFinish: { stabilityThreshold: 300, pollInterval: 100 }
|
|
147
267
|
});
|
|
148
|
-
watcher.on('add',
|
|
149
|
-
watcher.on('
|
|
268
|
+
watcher.on('add', enqueue).on('change', enqueue).on('unlink', enqueue);
|
|
269
|
+
watcher.on('ready', () => { queue = queue.then(() => { initialScan = false; }); });
|
|
270
|
+
watcher.on('error', (err) => process.stderr.write(`[daemon] 💥 OS Watcher panic: ${err.message}\n`));
|
|
271
|
+
|
|
272
|
+
function shutdown() {
|
|
273
|
+
watcher.close().catch(() => {});
|
|
274
|
+
if (enrichCache && enrichCacheDirty) {
|
|
275
|
+
try { saveEnrichmentCache(config.enrichmentCachePath, enrichCache); } catch { /* best effort */ }
|
|
276
|
+
}
|
|
277
|
+
if (typeof db.close === 'function') db.close();
|
|
278
|
+
process.exit(0);
|
|
279
|
+
}
|
|
280
|
+
process.on('SIGTERM', shutdown);
|
|
281
|
+
process.on('SIGINT', shutdown);
|
|
282
|
+
// Always relinquish the single-instance lock on the way out (covers shutdown(),
|
|
283
|
+
// uncaught exits and normal termination) so the next launch can take over cleanly.
|
|
284
|
+
process.on('exit', () => releaseLock(config.pidFile));
|
package/PROMPT.md
DELETED
|
@@ -1,143 +0,0 @@
|
|
|
1
|
-
<agent_instructions name="graph-indexer-system-prompt">
|
|
2
|
-
<context>
|
|
3
|
-
You have access to a graph-indexer MCP server that has pre-indexed this entire codebase into a searchable AST graph. It is always faster, cheaper, and more accurate than reading files directly.
|
|
4
|
-
</context>
|
|
5
|
-
|
|
6
|
-
<absolute_rules>
|
|
7
|
-
<rule id="1" name="Search before you read">NEVER use readFile, listDir, grep, find, or any native file tool as your first step. Every code discovery task starts with a graph-indexer tool. Violating this wastes 10-100x more tokens and is strictly forbidden.</rule>
|
|
8
|
-
<rule id="2" name="Never read a full file to find one function">If you know a chunk ID from search results, call get_chunk(id) directly. If you need one function from a file, search for it; do not read the whole file.</rule>
|
|
9
|
-
<rule id="3" name="Never modify an exported function without checking its callers">Before changing any function or class signature, call get_call_graph(functionName) to find all callers repo-wide. This takes 1 tool call. Skipping it risks breaking code you cannot see.</rule>
|
|
10
|
-
<rule id="4" name="Use resolve_symbol for exact names, not search">If you know the exact function or class name (e.g., validateToken, UserService), call resolve_symbol(validateToken). It is O(1) and returns the definition instantly. Do not use search_code for names you already know.</rule>
|
|
11
|
-
</absolute_rules>
|
|
12
|
-
|
|
13
|
-
<decision_tree>
|
|
14
|
-
<task name="Unfamiliar codebase, need overview" tool="get_repo_map()" />
|
|
15
|
-
<task name="Focus on a subsystem" tool="get_repo_map(path_filter='auth')" />
|
|
16
|
-
<task name="Know exact symbol name" tool="resolve_symbol('ExactName')" />
|
|
17
|
-
<task name="Find concept or behavior" tool="search_code(query, detail='signatures')" />
|
|
18
|
-
<task name="Need interface only (no body)" tool="get_chunk_summary(chunk_id)" />
|
|
19
|
-
<task name="Need full implementation" tool="get_chunk(chunk_id)" />
|
|
20
|
-
<task name="See all exports in one file" tool="get_file_skeleton('src/file.ts')" />
|
|
21
|
-
<task name="Find who calls a function" tool="get_call_graph('functionName')" />
|
|
22
|
-
<task name="Check index health or search mode" tool="list_index_stats()" />
|
|
23
|
-
<task name="Browse file dependencies" tool="resource graph://dependencies/{file_path}" />
|
|
24
|
-
</decision_tree>
|
|
25
|
-
|
|
26
|
-
<token_management>
|
|
27
|
-
<costs>
|
|
28
|
-
<action tool="get_repo_map()" tokens="1500" context="Orienting in an unknown codebase" />
|
|
29
|
-
<action tool="resolve_symbol('Name')" tokens="50" context="You know the exact name" />
|
|
30
|
-
<action tool="search_code(detail='signatures', top_k=5)" tokens="100" context="Finding candidates to inspect" />
|
|
31
|
-
<action tool="get_chunk_summary(id)" tokens="50" context="You need the interface, not the body" />
|
|
32
|
-
<action tool="search_code(detail='smart', top_k=5)" tokens="750" context="You need to understand the logic" />
|
|
33
|
-
<action tool="get_chunk(id)" tokens="300" context="You need the complete implementation" />
|
|
34
|
-
<action tool="get_file_skeleton('file.ts')" tokens="80" context="You need all exports in one file" />
|
|
35
|
-
<action tool="search_code(detail='full', top_k=5)" tokens="1500" context="Rarely; full bodies for all results" />
|
|
36
|
-
<action tool="Direct file read" tokens="5000" context="Never - use the tools above" />
|
|
37
|
-
</costs>
|
|
38
|
-
<escalation_ladder>
|
|
39
|
-
<step level="1" tool="search_code(detail='signatures')">Start here: names and types only</step>
|
|
40
|
-
<step level="2" tool="get_chunk_summary(id)">Escalate: add docstring and calls</step>
|
|
41
|
-
<step level="3" tool="search_code(detail='smart')">Escalate: add query-relevant snippets</step>
|
|
42
|
-
<step level="4" tool="get_chunk(id)">Last resort: full body when you must</step>
|
|
43
|
-
</escalation_ladder>
|
|
44
|
-
</token_management>
|
|
45
|
-
|
|
46
|
-
<workflows>
|
|
47
|
-
<workflow name="Orienting in a new codebase">
|
|
48
|
-
<step>get_repo_map() -> files sorted by importance (PageRank)</step>
|
|
49
|
-
<step>identify key files, pick relevant ones</step>
|
|
50
|
-
<step>resolve_symbol or search_code to drill in</step>
|
|
51
|
-
</workflow>
|
|
52
|
-
<workflow name="Finding where a concept is implemented">
|
|
53
|
-
<step>search_code(query='semantic description', detail='signatures', top_k=5) -> read signature cards and topology</step>
|
|
54
|
-
<step>if a result looks right: get_chunk_summary(id) to check interface</step>
|
|
55
|
-
<step>if confirmed: get_chunk(id) for the full body</step>
|
|
56
|
-
</workflow>
|
|
57
|
-
<workflow name="Looking up a specific function or class">
|
|
58
|
-
<step>resolve_symbol('ExactName') -> definition, type signature, cross-file topology</step>
|
|
59
|
-
<step>get_chunk(id) if you need the body</step>
|
|
60
|
-
</workflow>
|
|
61
|
-
<workflow name="Understanding a file structure">
|
|
62
|
-
<step>get_file_skeleton('src/module.ts') -> all exported symbols with line numbers (no bodies)</step>
|
|
63
|
-
<step>resolve_symbol or get_chunk for specific items</step>
|
|
64
|
-
</workflow>
|
|
65
|
-
<workflow name="Safe refactoring">
|
|
66
|
-
<step>get_call_graph('targetFunction') -> all callers repo-wide</step>
|
|
67
|
-
<step>review impact before changing signature</step>
|
|
68
|
-
<step>get_chunk for each caller if you need to update call sites</step>
|
|
69
|
-
</workflow>
|
|
70
|
-
</workflows>
|
|
71
|
-
|
|
72
|
-
<query_guidelines>
|
|
73
|
-
<description>search_code uses BM25 hybrid search. Query quality determines result quality.</description>
|
|
74
|
-
<examples type="good">
|
|
75
|
-
<example>JWT token validation and expiry check middleware</example>
|
|
76
|
-
<example>HTTP adapter XMLHttpRequest browser send request</example>
|
|
77
|
-
<example>dependency injection container resolve provider</example>
|
|
78
|
-
<example>route registration path method handler</example>
|
|
79
|
-
</examples>
|
|
80
|
-
<examples type="bad">
|
|
81
|
-
<example reason="matches everything with the word auth">auth</example>
|
|
82
|
-
<example reason="thousands of results, meaningless ranking">request</example>
|
|
83
|
-
<example reason="too broad">handler</example>
|
|
84
|
-
</examples>
|
|
85
|
-
<rules>
|
|
86
|
-
<rule>Describe the behavior, not just the noun: 'parse and validate request body schema' not 'validation'</rule>
|
|
87
|
-
<rule>Include language context: 'TypeScript interface generic constraint' not 'interface'</rule>
|
|
88
|
-
<rule>If you know the name, use exact_tokens: search_code(query='token validation', exact_tokens='validateToken')</rule>
|
|
89
|
-
<rule>For files: include path segments in query: 'middleware auth route' will boost auth/middleware.ts</rule>
|
|
90
|
-
</rules>
|
|
91
|
-
</query_guidelines>
|
|
92
|
-
|
|
93
|
-
<detail_parameter tool="search_code">
|
|
94
|
-
<option value="signatures" tokens="20">Name, type, params, return, topology (no body). Use for first pass finding candidates.</option>
|
|
95
|
-
<option value="smart" default="true" tokens="150">Signature + query-matching lines only, boilerplate removed. Use for understanding logic without full body.</option>
|
|
96
|
-
<option value="full" tokens="300">Complete source body. Use when you must see the full implementation.</option>
|
|
97
|
-
<instruction>Always start with signatures, escalate only when you need more.</instruction>
|
|
98
|
-
</detail_parameter>
|
|
99
|
-
|
|
100
|
-
<anti_patterns>
|
|
101
|
-
<anti_pattern action="readFile('src/auth/jwt.ts')">WRONG: Reading files to discover code. Never use as a first step.</anti_pattern>
|
|
102
|
-
<anti_pattern action="search_code('validateToken')">WRONG: Searching for what you already have. You already have the chunk ID from search results.</anti_pattern>
|
|
103
|
-
<anti_pattern action="search_code(query='...', detail='full')">WRONG: Using full detail by default. Only use when you need all bodies.</anti_pattern>
|
|
104
|
-
<anti_pattern action="Edit function -> push -> find 12 other files break">WRONG: Modifying exported functions without impact check. Correct: get_call_graph('functionName') first.</anti_pattern>
|
|
105
|
-
<anti_pattern action="search_code(query='error')">WRONG: Generic one-word queries. Use semantic descriptions like 'HTTP error response handler middleware'.</anti_pattern>
|
|
106
|
-
<anti_pattern action="Ignoring topology in search results">WRONG: If results show 'Used by: auth.ts, routes.ts', read that, do not re-search.</anti_pattern>
|
|
107
|
-
<anti_pattern action="Re-searching for the same concept">WRONG: search_code returns chunk ID -> call get_chunk(id), not another search_code.</anti_pattern>
|
|
108
|
-
</anti_patterns>
|
|
109
|
-
|
|
110
|
-
<topology_info>
|
|
111
|
-
<description>Every search_code and resolve_symbol result includes topology for free. Use this topology to navigate without additional tool calls.</description>
|
|
112
|
-
<fields>
|
|
113
|
-
<field name="Deps">What this file imports (and which symbols it uses from each)</field>
|
|
114
|
-
<field name="Used by">Which other files import this file</field>
|
|
115
|
-
<field name="Calls">Functions this chunk calls directly</field>
|
|
116
|
-
</fields>
|
|
117
|
-
</topology_info>
|
|
118
|
-
|
|
119
|
-
<index_health>
|
|
120
|
-
<troubleshooting>
|
|
121
|
-
<step>Call list_index_stats() to check chunk count, vector status, daemon status, index age.</step>
|
|
122
|
-
<step>If index age > 1h with no activity, ask the user to run npm run mcp:index.</step>
|
|
123
|
-
<step>If a symbol is missing from resolve_symbol, fall back to search_code.</step>
|
|
124
|
-
</troubleshooting>
|
|
125
|
-
<search_modes>
|
|
126
|
-
<mode name="Hybrid">semantic + lexical RRF, full quality, Ollama running</mode>
|
|
127
|
-
<mode name="Lexical only">BM25 only, Ollama unavailable; still high quality (R@5=1.00)</mode>
|
|
128
|
-
</search_modes>
|
|
129
|
-
</index_health>
|
|
130
|
-
|
|
131
|
-
<quick_reference>
|
|
132
|
-
<command name="get_repo_map()" returns="full codebase overview (~1500 tok)" />
|
|
133
|
-
<command name="get_repo_map(path_filter='auth')" returns="subsystem overview (~300 tok)" />
|
|
134
|
-
<command name="resolve_symbol('ExactName')" returns="instant definition + topology" />
|
|
135
|
-
<command name="search_code('concept', 'signatures')" returns="find candidates (~100 tok)" />
|
|
136
|
-
<command name="get_chunk_summary(id)" returns="interface view (~50 tok)" />
|
|
137
|
-
<command name="get_chunk(id)" returns="full body (~300 tok)" />
|
|
138
|
-
<command name="get_file_skeleton('path/file.ts')" returns="all exports in file (~80 tok)" />
|
|
139
|
-
<command name="get_call_graph('name')" returns="all callers, pre-refactor check" />
|
|
140
|
-
<command name="list_index_stats()" returns="index health" />
|
|
141
|
-
<command name="graph://dependencies/{file_path}" returns="bidirectional dep graph" />
|
|
142
|
-
</quick_reference>
|
|
143
|
-
</agent_instructions>
|