graph-indexer 2.1.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +260 -1
- package/README.md +232 -312
- package/SECURITY.md +37 -59
- package/bin/graph-indexer.mjs +251 -0
- package/integrations/claude-code/.claude-plugin/plugin.json +8 -0
- package/integrations/claude-code/.mcp.json +8 -0
- package/integrations/claude-code/agents/code-structure.md +22 -0
- package/integrations/claude-code/hooks/gi-hook.mjs +99 -0
- package/integrations/claude-code/hooks/hooks.json +16 -0
- package/integrations/opencode/graph-indexer.js +124 -0
- package/package.json +31 -105
- package/src/cli/helper.mjs +67 -0
- package/src/cli/hook.mjs +356 -0
- package/src/cli/init.mjs +255 -0
- package/src/cli/resident.mjs +175 -0
- package/src/index/discover.mjs +130 -0
- package/src/index/fingerprint.mjs +30 -0
- package/src/index/graph.mjs +61 -0
- package/src/index/indexer.mjs +476 -0
- package/src/index/modules.mjs +331 -0
- package/src/index/resolver.mjs +954 -0
- package/src/mcp/render.mjs +60 -0
- package/src/mcp/server.mjs +147 -0
- package/src/mcp/tools.mjs +973 -0
- package/src/parse/arity.mjs +152 -0
- package/src/parse/extract.mjs +1031 -0
- package/src/parse/lang/csharp.mjs +106 -0
- package/src/parse/lang/ecma.mjs +438 -0
- package/src/parse/lang/go.mjs +194 -0
- package/src/parse/lang/jvm.mjs +118 -0
- package/src/parse/lang/kotlin.mjs +97 -0
- package/src/parse/lang/misc.mjs +115 -0
- package/src/parse/lang/native.mjs +126 -0
- package/src/parse/lang/python.mjs +153 -0
- package/src/parse/lang/rust.mjs +157 -0
- package/src/parse/lang/scripting.mjs +219 -0
- package/src/parse/languages.mjs +48 -0
- package/src/parse/runtime.mjs +56 -0
- package/src/query/check.mjs +343 -0
- package/src/query/facts.mjs +323 -0
- package/src/query/intel.mjs +680 -0
- package/src/query/read.mjs +81 -0
- package/src/query/testcmd.mjs +119 -0
- package/src/query/textsearch.mjs +160 -0
- package/src/search/search.mjs +210 -0
- package/src/search/tokenize.mjs +188 -0
- package/src/store/db.mjs +169 -0
- package/src/util/paths.mjs +24 -0
- package/vendor/grammars/bash.wasm.gz +0 -0
- package/vendor/grammars/c.wasm.gz +0 -0
- package/vendor/grammars/c_sharp.wasm.gz +0 -0
- package/vendor/grammars/cpp.wasm.gz +0 -0
- package/vendor/grammars/css.wasm.gz +0 -0
- package/vendor/grammars/go.wasm.gz +0 -0
- package/vendor/grammars/java.wasm.gz +0 -0
- package/vendor/grammars/javascript.wasm.gz +0 -0
- package/vendor/grammars/kotlin.wasm.gz +0 -0
- package/vendor/grammars/manifest.json +148 -0
- package/vendor/grammars/php.wasm.gz +0 -0
- package/vendor/grammars/python.wasm.gz +0 -0
- package/vendor/grammars/ruby.wasm.gz +0 -0
- package/vendor/grammars/rust.wasm.gz +0 -0
- package/vendor/grammars/scala.wasm.gz +0 -0
- package/vendor/grammars/tsx.wasm.gz +0 -0
- package/vendor/grammars/typescript.wasm.gz +0 -0
- package/vendor/web-tree-sitter/web-tree-sitter.js +3980 -0
- package/vendor/web-tree-sitter/web-tree-sitter.wasm +0 -0
- package/cli-ui.mjs +0 -78
- package/colbert.mjs +0 -168
- package/config.mjs +0 -494
- package/daemon-ctl.mjs +0 -175
- package/daemon-lock.mjs +0 -69
- package/embedders/python/mlx_embed_server.py +0 -61
- package/embedders/python/requirements-mlx.txt +0 -5
- package/embedders/setup-mlx.mjs +0 -110
- package/embeddings.mjs +0 -571
- package/engine/binary.mjs +0 -437
- package/engine/memory.mjs +0 -1003
- package/engine/sqlite.mjs +0 -1136
- package/enrichment.mjs +0 -521
- package/git-signals.mjs +0 -248
- package/indexer.mjs +0 -546
- package/init.mjs +0 -1809
- package/layout.mjs +0 -123
- package/mcp/centrality.mjs +0 -117
- package/mcp/format.mjs +0 -61
- package/mcp/resolver.mjs +0 -116
- package/mcp/symbolgraph.mjs +0 -137
- package/mcp/taint.mjs +0 -225
- package/mcp/tools.mjs +0 -1582
- package/mcp/topology.mjs +0 -676
- package/mcp-server.mjs +0 -197
- package/parse/extractor.mjs +0 -628
- package/parse/imports.mjs +0 -365
- package/parse/interprocedural.mjs +0 -134
- package/parse/languages.mjs +0 -253
- package/parse/metadata.mjs +0 -701
- package/parse/routes.mjs +0 -327
- package/parse/scip.mjs +0 -391
- package/parse/taint-patterns.mjs +0 -119
- package/prompts/CORE.md +0 -91
- package/prompts/DOMAIN_TEMPLATE.md +0 -46
- package/prompts/INTEGRATION.md +0 -123
- package/prompts/frameworks/ANDROID.md +0 -17
- package/prompts/frameworks/ASPNET_CORE.md +0 -19
- package/prompts/frameworks/FASTAPI_DJANGO.md +0 -20
- package/prompts/frameworks/LARAVEL_SYMFONY.md +0 -22
- package/prompts/frameworks/NODE_EXPRESS_NESTJS.md +0 -19
- package/prompts/frameworks/RAILS.md +0 -20
- package/prompts/frameworks/REACT.md +0 -20
- package/prompts/frameworks/SPRING_BOOT.md +0 -20
- package/prompts/languages/BASH.md +0 -22
- package/prompts/languages/C.md +0 -21
- package/prompts/languages/CSHARP.md +0 -24
- package/prompts/languages/CSS_SCSS.md +0 -22
- package/prompts/languages/GO.md +0 -24
- package/prompts/languages/JAVA.md +0 -23
- package/prompts/languages/JAVASCRIPT_TYPESCRIPT.md +0 -24
- package/prompts/languages/KOTLIN.md +0 -24
- package/prompts/languages/PHP.md +0 -23
- package/prompts/languages/PYTHON.md +0 -23
- package/prompts/languages/RUBY.md +0 -23
- package/prompts/languages/RUST.md +0 -21
- package/prompts/languages/SWIFT.md +0 -21
- package/seal.mjs +0 -400
- package/search-core.mjs +0 -777
- package/search-sparse.mjs +0 -183
- package/storage.mjs +0 -73
- package/watch-daemon.mjs +0 -289
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Code-aware tokenization shared by indexing and querying.
|
|
3
|
+
*
|
|
4
|
+
* An identifier contributes its full lowercase form AND its sub-words, so `getUserById`
|
|
5
|
+
* matches the query "get user by id", the query "getUserById", and the query "user".
|
|
6
|
+
* Acronyms split correctly (`parseHTTPHeader` → parse, http, header). Stemming happens in
|
|
7
|
+
* SQLite's FTS5 porter tokenizer on top of these tokens.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const IDENT_RE = /[A-Za-z_$][A-Za-z0-9_$]*|[0-9]+/g;
|
|
11
|
+
|
|
12
|
+
/** Split one identifier into lowercase sub-words (camel/Pascal/snake/kebab/acronyms/digits). */
|
|
13
|
+
export function splitIdentifier(id) {
|
|
14
|
+
const parts = id
|
|
15
|
+
.replace(/[$_]+/g, ' ')
|
|
16
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
17
|
+
.replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
|
|
18
|
+
.replace(/([A-Za-z])([0-9])/g, '$1 $2')
|
|
19
|
+
.replace(/([0-9])([A-Za-z])/g, '$1 $2')
|
|
20
|
+
.toLowerCase()
|
|
21
|
+
.split(/\s+/)
|
|
22
|
+
.filter(Boolean);
|
|
23
|
+
return parts;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Identifiers repeat heavily across a code base: cache each one's token expansion.
|
|
27
|
+
const TOKEN_CACHE = new Map();
|
|
28
|
+
const TOKEN_CACHE_MAX = 200_000;
|
|
29
|
+
|
|
30
|
+
function identifierTokens(id) {
|
|
31
|
+
let t = TOKEN_CACHE.get(id);
|
|
32
|
+
if (t) return t;
|
|
33
|
+
if (/^[0-9]+$/.test(id)) t = id.length <= 6 ? [id] : [];
|
|
34
|
+
else {
|
|
35
|
+
const lower = id.toLowerCase().replace(/^[$_]+|[$_]+$/g, '');
|
|
36
|
+
const parts = splitIdentifier(id);
|
|
37
|
+
t = [];
|
|
38
|
+
if (lower.length >= 2 && (parts.length !== 1 || parts[0] !== lower)) t.push(lower.replace(/[$]/g, ''));
|
|
39
|
+
for (const p of parts) if (p.length >= 2 || parts.length === 1) t.push(p);
|
|
40
|
+
}
|
|
41
|
+
if (TOKEN_CACHE.size >= TOKEN_CACHE_MAX) TOKEN_CACHE.clear();
|
|
42
|
+
TOKEN_CACHE.set(id, t);
|
|
43
|
+
return t;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Tokens for indexing a piece of code/text (identifier forms + sub-words). */
|
|
47
|
+
export function codeTokens(text, { maxTokens = Infinity } = {}) {
|
|
48
|
+
const out = [];
|
|
49
|
+
if (!text) return out;
|
|
50
|
+
IDENT_RE.lastIndex = 0;
|
|
51
|
+
let m;
|
|
52
|
+
while ((m = IDENT_RE.exec(text)) && out.length < maxTokens) {
|
|
53
|
+
for (const t of identifierTokens(m[0])) out.push(t);
|
|
54
|
+
}
|
|
55
|
+
return out;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function tokenString(text, opts) { return codeTokens(text, opts).join(' '); }
|
|
59
|
+
|
|
60
|
+
// Words that carry no retrieval signal in a question about code.
|
|
61
|
+
export const STOPWORDS = new Set([
|
|
62
|
+
'a', 'an', 'the', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'at', 'by', 'for', 'with', 'from', 'into', 'onto',
|
|
63
|
+
'is', 'are', 'was', 'were', 'be', 'been', 'being', 'am', 'do', 'does', 'did', 'done', 'doing', 'have', 'has', 'had',
|
|
64
|
+
'it', 'its', 'this', 'that', 'these', 'those', 'there', 'here', 'what', 'which', 'who', 'whom', 'whose', 'where',
|
|
65
|
+
'when', 'why', 'how', 'can', 'could', 'should', 'would', 'will', 'shall', 'may', 'might', 'must', 'i', 'we', 'you',
|
|
66
|
+
'they', 'he', 'she', 'me', 'us', 'them', 'my', 'our', 'your', 'their', 'if', 'then', 'else', 'than', 'so', 'such',
|
|
67
|
+
'as', 'not', 'no', 'yes', 'all', 'any', 'each', 'every', 'some', 'most', 'more', 'other', 'about', 'over', 'under',
|
|
68
|
+
'up', 'down', 'out', 'off', 'again', 'further', 'once', 'also', 'just', 'only', 'very', 'too', 'via', 'per', 'like',
|
|
69
|
+
'code', 'function', 'functions', 'method', 'methods', 'class', 'classes', 'implementation', 'implemented', 'implement',
|
|
70
|
+
'implements', 'logic', 'where', 'find', 'show', 'defined', 'handled', 'handle', 'happens', 'used', 'using', 'use',
|
|
71
|
+
'get', 'gets', 'responsible', 'thing', 'things', 'part', 'called', 'call', 'calls', 'file', 'files', 'module',
|
|
72
|
+
]);
|
|
73
|
+
// NOTE: 'get', 'handle', 'find', 'use', 'call' are stopwords only for natural-language queries;
|
|
74
|
+
// identifier-shaped queries keep them (see analyzeQuery).
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Small, curated concept thesaurus for natural-language → code vocabulary gaps.
|
|
78
|
+
* Each group is symmetric; expansions are scored below literal terms.
|
|
79
|
+
*/
|
|
80
|
+
const CONCEPT_GROUPS = [
|
|
81
|
+
['auth', 'authentication', 'authenticate', 'authorization', 'authorize', 'login', 'credential', 'credentials'],
|
|
82
|
+
['db', 'database', 'sql', 'query', 'repository', 'repo', 'dao', 'store', 'persistence'],
|
|
83
|
+
['config', 'configuration', 'settings', 'options', 'opts', 'conf', 'cfg'],
|
|
84
|
+
['init', 'initialize', 'initialise', 'setup', 'bootstrap', 'boot', 'startup'],
|
|
85
|
+
['err', 'error', 'errors', 'exception', 'failure', 'fault'],
|
|
86
|
+
['req', 'request'], ['res', 'resp', 'response', 'reply'],
|
|
87
|
+
['msg', 'message'], ['ctx', 'context'], ['env', 'environment'], ['impl', 'implementation'],
|
|
88
|
+
['util', 'utils', 'utility', 'helper', 'helpers'], ['str', 'string', 'text'], ['num', 'number', 'count'],
|
|
89
|
+
['param', 'params', 'parameter', 'arg', 'args', 'argument'], ['fn', 'func', 'callback', 'cb', 'handler'],
|
|
90
|
+
['del', 'delete', 'remove', 'destroy', 'drop', 'erase'], ['create', 'new', 'make', 'build', 'construct', 'add', 'insert'],
|
|
91
|
+
['fetch', 'get', 'retrieve', 'load', 'read', 'lookup'], ['save', 'store', 'persist', 'write', 'put', 'commit'],
|
|
92
|
+
['update', 'modify', 'patch', 'edit', 'change', 'set'],
|
|
93
|
+
['search', 'find', 'query', 'lookup', 'filter', 'match'], ['list', 'all', 'collection', 'index'],
|
|
94
|
+
['parse', 'decode', 'deserialize', 'unmarshal', 'unmarshall', 'read'],
|
|
95
|
+
['serialize', 'encode', 'marshal', 'stringify', 'dump', 'render'],
|
|
96
|
+
['validate', 'validation', 'validator', 'check', 'verify', 'assert', 'ensure', 'sanitize'],
|
|
97
|
+
['send', 'emit', 'dispatch', 'publish', 'post', 'notify', 'broadcast'],
|
|
98
|
+
['receive', 'consume', 'listen', 'subscribe', 'on', 'handle'],
|
|
99
|
+
['route', 'routes', 'router', 'routing', 'endpoint', 'path', 'url', 'controller'],
|
|
100
|
+
['middleware', 'interceptor', 'hook', 'filter', 'guard', 'pipe', 'plugin'],
|
|
101
|
+
['cache', 'cached', 'memo', 'memoize', 'lru'], ['retry', 'retries', 'backoff', 'attempt'],
|
|
102
|
+
['timeout', 'deadline', 'expire', 'expiry', 'ttl'], ['log', 'logger', 'logging', 'trace', 'debug'],
|
|
103
|
+
['user', 'account', 'member', 'profile', 'owner'], ['permission', 'role', 'access', 'acl', 'policy', 'privilege'],
|
|
104
|
+
['token', 'jwt', 'session', 'cookie', 'bearer'], ['password', 'secret', 'hash', 'crypt', 'encrypt', 'cipher'],
|
|
105
|
+
['upload', 'file', 'attachment', 'blob', 'multipart'], ['schedule', 'cron', 'job', 'task', 'worker', 'queue'],
|
|
106
|
+
['event', 'listener', 'subscriber', 'observer', 'emitter', 'signal'], ['test', 'spec', 'mock', 'stub', 'fixture'],
|
|
107
|
+
['bind', 'binding', 'bound'], ['convert', 'transform', 'map', 'cast', 'coerce', 'normalize'],
|
|
108
|
+
['compare', 'equal', 'equals', 'diff', 'eq'], ['sort', 'order', 'rank', 'priority'],
|
|
109
|
+
['merge', 'combine', 'join', 'concat', 'extend', 'assign'], ['split', 'chunk', 'partition', 'segment'],
|
|
110
|
+
['wrap', 'wrapper', 'decorate', 'decorator', 'proxy', 'adapter'], ['escape', 'sanitize', 'encode', 'quote'],
|
|
111
|
+
['start', 'begin', 'launch', 'run', 'serve', 'listen'], ['stop', 'end', 'shutdown', 'close', 'terminate', 'dispose', 'teardown'],
|
|
112
|
+
['http', 'rest', 'api', 'client', 'fetch', 'axios'], ['header', 'headers'], ['body', 'payload', 'content', 'data'],
|
|
113
|
+
['redirect', 'forward', 'location'], ['status', 'code', 'state'], ['json', 'jsonify'], ['xml', 'html', 'template', 'view'],
|
|
114
|
+
['stream', 'pipe', 'buffer', 'reader', 'writer'], ['async', 'await', 'promise', 'future', 'deferred'],
|
|
115
|
+
['thread', 'lock', 'mutex', 'concurrent', 'sync'], ['socket', 'websocket', 'ws', 'connection', 'conn'],
|
|
116
|
+
['cors', 'origin', 'crossorigin'], ['cookie', 'cookies'], ['static', 'asset', 'assets', 'public'],
|
|
117
|
+
['dependency', 'dependencies', 'inject', 'injection', 'injectable', 'provider', 'container', 'di'],
|
|
118
|
+
['schema', 'model', 'entity', 'dto', 'type'], ['migrate', 'migration', 'schema'], ['email', 'mail', 'smtp'],
|
|
119
|
+
['payment', 'charge', 'billing', 'invoice', 'stripe'], ['image', 'img', 'picture', 'photo', 'thumbnail'],
|
|
120
|
+
['date', 'time', 'timestamp', 'datetime', 'clock'], ['random', 'rand', 'uuid', 'id', 'identifier'],
|
|
121
|
+
['env', 'environment', 'variable', 'var'], ['command', 'cmd', 'cli', 'arg', 'flag'],
|
|
122
|
+
['proxy', 'forward', 'tunnel'], ['compress', 'gzip', 'zip', 'deflate'], ['version', 'semver'],
|
|
123
|
+
];
|
|
124
|
+
|
|
125
|
+
const CONCEPTS = new Map();
|
|
126
|
+
for (const group of CONCEPT_GROUPS) {
|
|
127
|
+
for (const w of group) {
|
|
128
|
+
const set = CONCEPTS.get(w) ?? new Set();
|
|
129
|
+
for (const o of group) if (o !== w) set.add(o);
|
|
130
|
+
CONCEPTS.set(w, set);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function expandConcept(word) { return CONCEPTS.get(word) ?? null; }
|
|
135
|
+
|
|
136
|
+
const IDENTIFIER_SHAPE = /^(?:[a-z]+(?:[A-Z][a-z0-9]*)+|[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)+|[A-Za-z0-9]+(?:_[A-Za-z0-9]+)+|[A-Z][A-Z0-9_]{2,}|[A-Za-z_$][\w$]*(?:\.|::|#|->)[A-Za-z_$][\w$.:#>-]*|[A-Za-z_$][\w$]*\(\))$/;
|
|
137
|
+
|
|
138
|
+
/** Mixed-case (APIRoute, HTTPBearer, iOS), snake/kebab-free words with inner capitals, or a
|
|
139
|
+
* capitalized word after the first position ("... using Promise") read as code identifiers. */
|
|
140
|
+
function looksLikeIdentifier(t, position) {
|
|
141
|
+
if (!/^[A-Za-z_$][\w$]*$/.test(t)) return false;
|
|
142
|
+
if (/[A-Z]/.test(t.slice(1)) && /[a-z]/.test(t)) return true;
|
|
143
|
+
if (/[_$]/.test(t) || /[a-z][0-9]|[0-9][a-z]/i.test(t) && /[a-z]/i.test(t) && t.length > 2) return true;
|
|
144
|
+
if (position > 0 && /^[A-Z][a-z0-9]+$/.test(t)) return true;
|
|
145
|
+
return false;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Analyse a search query.
|
|
150
|
+
* @returns {{ raw, identifiers: string[], words: string[], terms: string[], expansions: string[], natural: boolean, filters: object }}
|
|
151
|
+
*/
|
|
152
|
+
export function analyzeQuery(query) {
|
|
153
|
+
const filters = {};
|
|
154
|
+
let q = String(query).replace(/\b(path|file|in|kind|lang|language):(\S+)/gi, (_, k, v) => {
|
|
155
|
+
const key = k.toLowerCase() === 'file' || k.toLowerCase() === 'in' ? 'path' : k.toLowerCase() === 'language' ? 'lang' : k.toLowerCase();
|
|
156
|
+
(filters[key] ??= []).push(v);
|
|
157
|
+
return ' ';
|
|
158
|
+
});
|
|
159
|
+
q = q.trim();
|
|
160
|
+
const rawTokens = q.split(/[\s,;]+/).filter(Boolean).map(t => t.replace(/^[`'"(]+|[`'"),.?!:]+$/g, '')).filter(Boolean);
|
|
161
|
+
const identifiers = [];
|
|
162
|
+
const words = [];
|
|
163
|
+
rawTokens.forEach((t, i) => {
|
|
164
|
+
if (IDENTIFIER_SHAPE.test(t) || looksLikeIdentifier(t, i)) identifiers.push(t.replace(/\(\)$/, ''));
|
|
165
|
+
else words.push(t);
|
|
166
|
+
});
|
|
167
|
+
const lowerWords = words.map(w => w.toLowerCase());
|
|
168
|
+
const stopCount = lowerWords.filter(w => STOPWORDS.has(w)).length;
|
|
169
|
+
// Natural language = several words, mostly not identifier-shaped, with function words.
|
|
170
|
+
const natural = rawTokens.length >= 3 && identifiers.length <= rawTokens.length / 2 && (stopCount >= 1 || rawTokens.length >= 5);
|
|
171
|
+
const termSet = new Set();
|
|
172
|
+
for (const id of identifiers) for (const t of codeTokens(id)) termSet.add(t);
|
|
173
|
+
for (const w of lowerWords) {
|
|
174
|
+
if (natural && STOPWORDS.has(w)) continue;
|
|
175
|
+
for (const t of codeTokens(w)) termSet.add(t);
|
|
176
|
+
}
|
|
177
|
+
// a single bare word query ("router") is an identifier lookup as much as a word
|
|
178
|
+
if (!identifiers.length && rawTokens.length <= 2) for (const w of rawTokens) if (/^[A-Za-z_$][\w$]*$/.test(w)) identifiers.push(w);
|
|
179
|
+
const terms = [...termSet].filter(t => t.length >= 2 || /^[a-z]$/.test(t) === false);
|
|
180
|
+
const expansions = new Set();
|
|
181
|
+
if (natural || identifiers.length === 0) {
|
|
182
|
+
for (const t of terms) {
|
|
183
|
+
const e = expandConcept(t);
|
|
184
|
+
if (e) for (const x of e) if (!termSet.has(x)) expansions.add(x);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return { raw: q, identifiers, words: lowerWords, terms, expansions: [...expansions], natural, filters };
|
|
188
|
+
}
|
package/src/store/db.mjs
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Persistent index store (node:sqlite, WAL). Every fact is owned by exactly one file, so
|
|
3
|
+
* re-indexing a file is "delete its rows, insert the new ones" — the Glean model, which keeps
|
|
4
|
+
* incremental updates simple and correct.
|
|
5
|
+
*/
|
|
6
|
+
import fs from 'node:fs';
|
|
7
|
+
import path from 'node:path';
|
|
8
|
+
|
|
9
|
+
// node:sqlite prints an ExperimentalWarning on Node 22; it is noise on an MCP server's stderr.
|
|
10
|
+
const origEmit = process.emitWarning;
|
|
11
|
+
process.emitWarning = function (warning, ...rest) {
|
|
12
|
+
const msg = typeof warning === 'string' ? warning : warning?.message;
|
|
13
|
+
if (msg && msg.includes('SQLite is an experimental feature')) return;
|
|
14
|
+
return origEmit.call(this, warning, ...rest);
|
|
15
|
+
};
|
|
16
|
+
const { DatabaseSync } = await import('node:sqlite');
|
|
17
|
+
|
|
18
|
+
export const SCHEMA_VERSION = 9;
|
|
19
|
+
|
|
20
|
+
const SCHEMA = `
|
|
21
|
+
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
|
|
22
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
23
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
24
|
+
path TEXT NOT NULL UNIQUE,
|
|
25
|
+
lang TEXT NOT NULL,
|
|
26
|
+
size INTEGER NOT NULL,
|
|
27
|
+
mtime_ms INTEGER NOT NULL,
|
|
28
|
+
hash TEXT NOT NULL,
|
|
29
|
+
lines INTEGER NOT NULL,
|
|
30
|
+
package TEXT,
|
|
31
|
+
is_test INTEGER NOT NULL DEFAULT 0,
|
|
32
|
+
parse_errors INTEGER NOT NULL DEFAULT 0,
|
|
33
|
+
indexed_at INTEGER NOT NULL
|
|
34
|
+
);
|
|
35
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
36
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT, -- ids are never reused: stable ids of re-indexed symbols are re-inserted explicitly
|
|
37
|
+
file_id INTEGER NOT NULL,
|
|
38
|
+
name TEXT NOT NULL,
|
|
39
|
+
name_lc TEXT NOT NULL,
|
|
40
|
+
qname TEXT NOT NULL,
|
|
41
|
+
kind TEXT NOT NULL,
|
|
42
|
+
parent_id INTEGER,
|
|
43
|
+
owner TEXT,
|
|
44
|
+
start_line INTEGER NOT NULL, start_col INTEGER NOT NULL,
|
|
45
|
+
end_line INTEGER NOT NULL, end_col INTEGER NOT NULL,
|
|
46
|
+
name_line INTEGER NOT NULL, name_col INTEGER NOT NULL,
|
|
47
|
+
sig TEXT, doc TEXT, type TEXT,
|
|
48
|
+
exported INTEGER NOT NULL DEFAULT 0,
|
|
49
|
+
visibility TEXT,
|
|
50
|
+
decorators TEXT,
|
|
51
|
+
bases TEXT,
|
|
52
|
+
ordinal INTEGER NOT NULL DEFAULT 0,
|
|
53
|
+
is_static INTEGER NOT NULL DEFAULT 0
|
|
54
|
+
);
|
|
55
|
+
CREATE INDEX IF NOT EXISTS symbols_file ON symbols(file_id);
|
|
56
|
+
CREATE INDEX IF NOT EXISTS symbols_name ON symbols(name_lc);
|
|
57
|
+
CREATE INDEX IF NOT EXISTS symbols_qname ON symbols(qname);
|
|
58
|
+
CREATE INDEX IF NOT EXISTS symbols_parent ON symbols(parent_id);
|
|
59
|
+
CREATE TABLE IF NOT EXISTS refs (
|
|
60
|
+
id INTEGER PRIMARY KEY,
|
|
61
|
+
file_id INTEGER NOT NULL,
|
|
62
|
+
src_id INTEGER,
|
|
63
|
+
name TEXT NOT NULL,
|
|
64
|
+
kind TEXT NOT NULL,
|
|
65
|
+
line INTEGER NOT NULL,
|
|
66
|
+
col INTEGER NOT NULL,
|
|
67
|
+
recv TEXT,
|
|
68
|
+
recv_type TEXT,
|
|
69
|
+
dst_id INTEGER,
|
|
70
|
+
conf REAL NOT NULL DEFAULT 0,
|
|
71
|
+
ncand INTEGER NOT NULL DEFAULT 0,
|
|
72
|
+
argc INTEGER -- arguments of a call or construction (-1: a spread makes it a lower bound), else NULL
|
|
73
|
+
);
|
|
74
|
+
CREATE INDEX IF NOT EXISTS refs_file ON refs(file_id);
|
|
75
|
+
CREATE INDEX IF NOT EXISTS refs_dst ON refs(dst_id);
|
|
76
|
+
CREATE INDEX IF NOT EXISTS refs_src ON refs(src_id);
|
|
77
|
+
CREATE INDEX IF NOT EXISTS refs_name ON refs(name);
|
|
78
|
+
CREATE TABLE IF NOT EXISTS imports (
|
|
79
|
+
file_id INTEGER NOT NULL,
|
|
80
|
+
source TEXT,
|
|
81
|
+
imported TEXT,
|
|
82
|
+
local TEXT,
|
|
83
|
+
reexport TEXT,
|
|
84
|
+
wildcard INTEGER NOT NULL DEFAULT 0,
|
|
85
|
+
line INTEGER,
|
|
86
|
+
target_file_id INTEGER,
|
|
87
|
+
target_dir TEXT,
|
|
88
|
+
target_path TEXT
|
|
89
|
+
);
|
|
90
|
+
CREATE INDEX IF NOT EXISTS imports_file ON imports(file_id);
|
|
91
|
+
CREATE INDEX IF NOT EXISTS imports_target ON imports(target_file_id);
|
|
92
|
+
CREATE TABLE IF NOT EXISTS fields (
|
|
93
|
+
file_id INTEGER NOT NULL,
|
|
94
|
+
owner_qname TEXT NOT NULL,
|
|
95
|
+
name TEXT NOT NULL,
|
|
96
|
+
type TEXT NOT NULL
|
|
97
|
+
);
|
|
98
|
+
CREATE INDEX IF NOT EXISTS fields_file ON fields(file_id);
|
|
99
|
+
CREATE INDEX IF NOT EXISTS fields_owner ON fields(owner_qname);
|
|
100
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(
|
|
101
|
+
name, qname, sig, doc, path, body,
|
|
102
|
+
tokenize = 'porter unicode61 remove_diacritics 2',
|
|
103
|
+
content = '', contentless_delete = 1
|
|
104
|
+
);
|
|
105
|
+
-- one document per file (rowid = files.id): file-level relevance for hierarchical ranking
|
|
106
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts5(
|
|
107
|
+
path, body,
|
|
108
|
+
tokenize = 'porter unicode61 remove_diacritics 2',
|
|
109
|
+
content = '', contentless_delete = 1
|
|
110
|
+
);
|
|
111
|
+
`;
|
|
112
|
+
|
|
113
|
+
export class Store {
|
|
114
|
+
constructor(dbPath) {
|
|
115
|
+
this.path = dbPath;
|
|
116
|
+
this.stmts = new Map();
|
|
117
|
+
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
118
|
+
this.db = new DatabaseSync(dbPath);
|
|
119
|
+
this.db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA temp_store = MEMORY; PRAGMA cache_size = -65536; PRAGMA busy_timeout = 5000;');
|
|
120
|
+
const v = this.#schemaVersion();
|
|
121
|
+
if (v !== null && v !== SCHEMA_VERSION) {
|
|
122
|
+
// schema changed: rebuild from scratch (the index is a cache of the source tree)
|
|
123
|
+
this.db.close();
|
|
124
|
+
for (const suffix of ['', '-wal', '-shm']) { try { fs.unlinkSync(dbPath + suffix); } catch { /* none */ } }
|
|
125
|
+
this.db = new DatabaseSync(dbPath);
|
|
126
|
+
this.db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA temp_store = MEMORY; PRAGMA cache_size = -65536; PRAGMA busy_timeout = 5000;');
|
|
127
|
+
}
|
|
128
|
+
this.db.exec(SCHEMA);
|
|
129
|
+
this.setMeta('schema_version', String(SCHEMA_VERSION));
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
#schemaVersion() {
|
|
133
|
+
try {
|
|
134
|
+
const row = this.db.prepare("SELECT value FROM meta WHERE key = 'schema_version'").get();
|
|
135
|
+
return row ? Number(row.value) : null;
|
|
136
|
+
} catch { return null; }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Cached prepared statement. */
|
|
140
|
+
q(sql) {
|
|
141
|
+
let s = this.stmts.get(sql);
|
|
142
|
+
if (!s) { s = this.db.prepare(sql); this.stmts.set(sql, s); }
|
|
143
|
+
return s;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
all(sql, ...params) { return this.q(sql).all(...params); }
|
|
147
|
+
get(sql, ...params) { return this.q(sql).get(...params); }
|
|
148
|
+
run(sql, ...params) { return this.q(sql).run(...params); }
|
|
149
|
+
|
|
150
|
+
tx(fn) {
|
|
151
|
+
this.db.exec('BEGIN');
|
|
152
|
+
try { const r = fn(); this.db.exec('COMMIT'); return r; }
|
|
153
|
+
catch (e) { try { this.db.exec('ROLLBACK'); } catch { /* already rolled back */ } throw e; }
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Drop every indexed fact but keep the database (the index is a cache of the source tree). */
|
|
157
|
+
reset() {
|
|
158
|
+
this.tx(() => {
|
|
159
|
+
for (const t of ['files', 'symbols', 'refs', 'imports', 'fields']) this.db.exec(`DELETE FROM ${t}`);
|
|
160
|
+
this.db.exec("INSERT INTO fts(fts) VALUES ('delete-all')");
|
|
161
|
+
this.db.exec("INSERT INTO file_fts(file_fts) VALUES ('delete-all')");
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
getMeta(key) { return this.get('SELECT value FROM meta WHERE key = ?', key)?.value ?? null; }
|
|
166
|
+
setMeta(key, value) { this.run('INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value', key, value); }
|
|
167
|
+
|
|
168
|
+
close() { try { this.db.close(); } catch { /* closed */ } }
|
|
169
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Data layout: everything graph-indexer writes lives in <repo>/.graph-indexer/ (git-ignored). */
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
export const DATA_DIR_NAME = '.graph-indexer';
|
|
6
|
+
|
|
7
|
+
export function dataDir(root) {
|
|
8
|
+
const dir = path.join(root, DATA_DIR_NAME);
|
|
9
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
10
|
+
const gi = path.join(dir, '.gitignore');
|
|
11
|
+
if (!fs.existsSync(gi)) { try { fs.writeFileSync(gi, '*\n'); } catch { /* read-only */ } }
|
|
12
|
+
return dir;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Walk up from `start` to the nearest directory containing .git (or return start). */
|
|
16
|
+
export function findRepoRoot(start) {
|
|
17
|
+
let dir = path.resolve(start);
|
|
18
|
+
for (;;) {
|
|
19
|
+
if (fs.existsSync(path.join(dir, '.git'))) return dir;
|
|
20
|
+
const parent = path.dirname(dir);
|
|
21
|
+
if (parent === dir) return path.resolve(start);
|
|
22
|
+
dir = parent;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
{
|
|
2
|
+
"runtime": {
|
|
3
|
+
"package": "web-tree-sitter",
|
|
4
|
+
"version": "0.25.10",
|
|
5
|
+
"source": "https://github.com/tree-sitter/tree-sitter/releases/tag/v0.25.10",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"files": {
|
|
8
|
+
"web-tree-sitter.js": {
|
|
9
|
+
"sha256": "d4fc466df6358055253bc4cdaab23cdf84fe00741fa5a4ff56506c06c533da42",
|
|
10
|
+
"bytes": 150999
|
|
11
|
+
},
|
|
12
|
+
"web-tree-sitter.wasm": {
|
|
13
|
+
"sha256": "f38dcc4b43b818f9a0785bc1c6d5611a75ac4cdd428ff3f02757c34ca4e46d7f",
|
|
14
|
+
"bytes": 205488
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"grammars": {
|
|
19
|
+
"bash": {
|
|
20
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-bash",
|
|
21
|
+
"tag": "v0.25.1",
|
|
22
|
+
"asset": "tree-sitter-bash.wasm",
|
|
23
|
+
"sha256": "8292919c88a0f7d3fb31d0cd0253ca5a9531bc1ede82b0537f2c63dd8abe6a7a",
|
|
24
|
+
"bytes": 1358224,
|
|
25
|
+
"license": "MIT"
|
|
26
|
+
},
|
|
27
|
+
"c": {
|
|
28
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-c",
|
|
29
|
+
"tag": "v0.24.2",
|
|
30
|
+
"asset": "tree-sitter-c.wasm",
|
|
31
|
+
"sha256": "83e8d7902b9d7f8c7c5cd4bd9acb5c7eb5faf42c09f85546b183964d3b5f48f9",
|
|
32
|
+
"bytes": 645157,
|
|
33
|
+
"license": "MIT"
|
|
34
|
+
},
|
|
35
|
+
"c_sharp": {
|
|
36
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-c-sharp",
|
|
37
|
+
"tag": "v0.23.5",
|
|
38
|
+
"asset": "tree-sitter-c_sharp.wasm",
|
|
39
|
+
"sha256": "6f69e1cae44e1c32c1eccc170dc5a9778fb94ff716f71113fe1f8c4299aa2f40",
|
|
40
|
+
"bytes": 5350581,
|
|
41
|
+
"license": "MIT"
|
|
42
|
+
},
|
|
43
|
+
"cpp": {
|
|
44
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-cpp",
|
|
45
|
+
"tag": "v0.23.4",
|
|
46
|
+
"asset": "tree-sitter-cpp.wasm",
|
|
47
|
+
"sha256": "174eb0deb75b2ec7881bcacda9f995648d8e683956e5c2267e69ab6dc503fcbf",
|
|
48
|
+
"bytes": 3434931,
|
|
49
|
+
"license": "MIT"
|
|
50
|
+
},
|
|
51
|
+
"css": {
|
|
52
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-css",
|
|
53
|
+
"tag": "v0.25.0",
|
|
54
|
+
"asset": "tree-sitter-css.wasm",
|
|
55
|
+
"sha256": "8a23977fe271357cce6f254ef88c9bebf3854602d8046605aef6a45c02135c59",
|
|
56
|
+
"bytes": 128668,
|
|
57
|
+
"license": "MIT"
|
|
58
|
+
},
|
|
59
|
+
"go": {
|
|
60
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-go",
|
|
61
|
+
"tag": "v0.25.0",
|
|
62
|
+
"asset": "tree-sitter-go.wasm",
|
|
63
|
+
"sha256": "9504573f352b20be7f2f1911754d710622aedc15afff16d5ed8fb5645681aee7",
|
|
64
|
+
"bytes": 217182,
|
|
65
|
+
"license": "MIT"
|
|
66
|
+
},
|
|
67
|
+
"java": {
|
|
68
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-java",
|
|
69
|
+
"tag": "v0.23.5",
|
|
70
|
+
"asset": "tree-sitter-java.wasm",
|
|
71
|
+
"sha256": "4fdeac4ca6ca089f06c6f7e562abcac1733cd465728cc7031ebb73c2019122c4",
|
|
72
|
+
"bytes": 414641,
|
|
73
|
+
"license": "MIT"
|
|
74
|
+
},
|
|
75
|
+
"javascript": {
|
|
76
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-javascript",
|
|
77
|
+
"tag": "v0.25.0",
|
|
78
|
+
"asset": "tree-sitter-javascript.wasm",
|
|
79
|
+
"sha256": "5fb488d0cabb4775a594bab85682de5ad6ce83c0d6ac997a9f82dd084d571240",
|
|
80
|
+
"bytes": 411770,
|
|
81
|
+
"license": "MIT"
|
|
82
|
+
},
|
|
83
|
+
"kotlin": {
|
|
84
|
+
"repo": "https://github.com/fwcd/tree-sitter-kotlin",
|
|
85
|
+
"tag": "0.3.8",
|
|
86
|
+
"asset": "tree-sitter-kotlin.wasm",
|
|
87
|
+
"sha256": "c624e7443b371c28adc5d81674e73067564c12555ebe3ed96a6c8db814b7602d",
|
|
88
|
+
"bytes": 4052625,
|
|
89
|
+
"license": "MIT"
|
|
90
|
+
},
|
|
91
|
+
"php": {
|
|
92
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-php",
|
|
93
|
+
"tag": "v0.24.2",
|
|
94
|
+
"asset": "tree-sitter-php.wasm",
|
|
95
|
+
"sha256": "d4df6a6ff08c87c3ec4f9cbb785fe09998a0cb570e03f57d7b19b3acfb146aa7",
|
|
96
|
+
"bytes": 1058041,
|
|
97
|
+
"license": "MIT"
|
|
98
|
+
},
|
|
99
|
+
"python": {
|
|
100
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-python",
|
|
101
|
+
"tag": "v0.25.0",
|
|
102
|
+
"asset": "tree-sitter-python.wasm",
|
|
103
|
+
"sha256": "16108b50df4ee9a30168794252ab55e7c93bfc5765d7fa0aa3e335752c515f47",
|
|
104
|
+
"bytes": 457883,
|
|
105
|
+
"license": "MIT"
|
|
106
|
+
},
|
|
107
|
+
"ruby": {
|
|
108
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-ruby",
|
|
109
|
+
"tag": "v0.23.1",
|
|
110
|
+
"asset": "tree-sitter-ruby.wasm",
|
|
111
|
+
"sha256": "09a96427d7c72f0613ed470cd9812223fc4a91d6a9c025c0235cc6bd59ff96f4",
|
|
112
|
+
"bytes": 2106352,
|
|
113
|
+
"license": "MIT"
|
|
114
|
+
},
|
|
115
|
+
"rust": {
|
|
116
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-rust",
|
|
117
|
+
"tag": "v0.24.2",
|
|
118
|
+
"asset": "tree-sitter-rust.wasm",
|
|
119
|
+
"sha256": "24c89bd9252255e4aebbcbd7d2d308bd92c86dd95a130fdc80efa49577b8d738",
|
|
120
|
+
"bytes": 1118099,
|
|
121
|
+
"license": "MIT"
|
|
122
|
+
},
|
|
123
|
+
"scala": {
|
|
124
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-scala",
|
|
125
|
+
"tag": "v0.26.2",
|
|
126
|
+
"asset": "tree-sitter-scala.wasm",
|
|
127
|
+
"sha256": "37d7fe5a91ca98941dc05493b0c05a0df0f36df5035890fa00b02497c68aaac3",
|
|
128
|
+
"bytes": 3973559,
|
|
129
|
+
"license": "MIT"
|
|
130
|
+
},
|
|
131
|
+
"tsx": {
|
|
132
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-typescript",
|
|
133
|
+
"tag": "v0.23.2",
|
|
134
|
+
"asset": "tree-sitter-tsx.wasm",
|
|
135
|
+
"sha256": "79e5da75ea62855a0cd67177685f0164eac87d5f630b3cbe1e0a099751ad30f8",
|
|
136
|
+
"bytes": 1445638,
|
|
137
|
+
"license": "MIT"
|
|
138
|
+
},
|
|
139
|
+
"typescript": {
|
|
140
|
+
"repo": "https://github.com/tree-sitter/tree-sitter-typescript",
|
|
141
|
+
"tag": "v0.23.2",
|
|
142
|
+
"asset": "tree-sitter-typescript.wasm",
|
|
143
|
+
"sha256": "778025db5a8be0e70f8ccc3671e486dfeddd048c25d9e8a70c26de2e1bf6f97d",
|
|
144
|
+
"bytes": 1413849,
|
|
145
|
+
"license": "MIT"
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|