graph-indexer 2.1.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +260 -1
- package/README.md +232 -312
- package/SECURITY.md +37 -59
- package/bin/graph-indexer.mjs +251 -0
- package/integrations/claude-code/.claude-plugin/plugin.json +8 -0
- package/integrations/claude-code/.mcp.json +8 -0
- package/integrations/claude-code/agents/code-structure.md +22 -0
- package/integrations/claude-code/hooks/gi-hook.mjs +99 -0
- package/integrations/claude-code/hooks/hooks.json +16 -0
- package/integrations/opencode/graph-indexer.js +124 -0
- package/package.json +32 -109
- package/src/cli/helper.mjs +67 -0
- package/src/cli/hook.mjs +356 -0
- package/src/cli/init.mjs +255 -0
- package/src/cli/resident.mjs +175 -0
- package/src/index/discover.mjs +130 -0
- package/src/index/fingerprint.mjs +30 -0
- package/src/index/graph.mjs +61 -0
- package/src/index/indexer.mjs +476 -0
- package/src/index/modules.mjs +331 -0
- package/src/index/resolver.mjs +954 -0
- package/src/mcp/render.mjs +60 -0
- package/src/mcp/server.mjs +147 -0
- package/src/mcp/tools.mjs +973 -0
- package/src/parse/arity.mjs +152 -0
- package/src/parse/extract.mjs +1031 -0
- package/src/parse/lang/csharp.mjs +106 -0
- package/src/parse/lang/ecma.mjs +438 -0
- package/src/parse/lang/go.mjs +194 -0
- package/src/parse/lang/jvm.mjs +118 -0
- package/src/parse/lang/kotlin.mjs +97 -0
- package/src/parse/lang/misc.mjs +115 -0
- package/src/parse/lang/native.mjs +126 -0
- package/src/parse/lang/python.mjs +153 -0
- package/src/parse/lang/rust.mjs +157 -0
- package/src/parse/lang/scripting.mjs +219 -0
- package/src/parse/languages.mjs +48 -0
- package/src/parse/runtime.mjs +56 -0
- package/src/query/check.mjs +343 -0
- package/src/query/facts.mjs +323 -0
- package/src/query/intel.mjs +680 -0
- package/src/query/read.mjs +81 -0
- package/src/query/testcmd.mjs +119 -0
- package/src/query/textsearch.mjs +160 -0
- package/src/search/search.mjs +210 -0
- package/src/search/tokenize.mjs +188 -0
- package/src/store/db.mjs +169 -0
- package/src/util/paths.mjs +24 -0
- package/vendor/grammars/bash.wasm.gz +0 -0
- package/vendor/grammars/c.wasm.gz +0 -0
- package/vendor/grammars/c_sharp.wasm.gz +0 -0
- package/vendor/grammars/cpp.wasm.gz +0 -0
- package/vendor/grammars/css.wasm.gz +0 -0
- package/vendor/grammars/go.wasm.gz +0 -0
- package/vendor/grammars/java.wasm.gz +0 -0
- package/vendor/grammars/javascript.wasm.gz +0 -0
- package/vendor/grammars/kotlin.wasm.gz +0 -0
- package/vendor/grammars/manifest.json +148 -0
- package/vendor/grammars/php.wasm.gz +0 -0
- package/vendor/grammars/python.wasm.gz +0 -0
- package/vendor/grammars/ruby.wasm.gz +0 -0
- package/vendor/grammars/rust.wasm.gz +0 -0
- package/vendor/grammars/scala.wasm.gz +0 -0
- package/vendor/grammars/tsx.wasm.gz +0 -0
- package/vendor/grammars/typescript.wasm.gz +0 -0
- package/vendor/web-tree-sitter/web-tree-sitter.js +3980 -0
- package/vendor/web-tree-sitter/web-tree-sitter.wasm +0 -0
- package/cli-ui.mjs +0 -78
- package/colbert.mjs +0 -168
- package/config.mjs +0 -494
- package/daemon-ctl.mjs +0 -175
- package/daemon-lock.mjs +0 -69
- package/embedders/python/mlx_embed_server.py +0 -61
- package/embedders/python/requirements-mlx.txt +0 -5
- package/embedders/setup-mlx.mjs +0 -110
- package/embeddings.mjs +0 -571
- package/engine/binary.mjs +0 -437
- package/engine/memory.mjs +0 -1003
- package/engine/sqlite.mjs +0 -1136
- package/enrichment.mjs +0 -521
- package/git-signals.mjs +0 -248
- package/indexer.mjs +0 -546
- package/init.mjs +0 -1809
- package/layout.mjs +0 -123
- package/mcp/centrality.mjs +0 -117
- package/mcp/format.mjs +0 -61
- package/mcp/resolver.mjs +0 -116
- package/mcp/symbolgraph.mjs +0 -137
- package/mcp/taint.mjs +0 -225
- package/mcp/tools.mjs +0 -1582
- package/mcp/topology.mjs +0 -676
- package/mcp-server.mjs +0 -197
- package/parse/extractor.mjs +0 -628
- package/parse/imports.mjs +0 -365
- package/parse/interprocedural.mjs +0 -134
- package/parse/languages.mjs +0 -253
- package/parse/metadata.mjs +0 -701
- package/parse/routes.mjs +0 -327
- package/parse/scip.mjs +0 -391
- package/parse/taint-patterns.mjs +0 -119
- package/prompts/CORE.md +0 -91
- package/prompts/DOMAIN_TEMPLATE.md +0 -46
- package/prompts/INTEGRATION.md +0 -123
- package/prompts/frameworks/ANDROID.md +0 -17
- package/prompts/frameworks/ASPNET_CORE.md +0 -19
- package/prompts/frameworks/FASTAPI_DJANGO.md +0 -20
- package/prompts/frameworks/LARAVEL_SYMFONY.md +0 -22
- package/prompts/frameworks/NODE_EXPRESS_NESTJS.md +0 -19
- package/prompts/frameworks/RAILS.md +0 -20
- package/prompts/frameworks/REACT.md +0 -20
- package/prompts/frameworks/SPRING_BOOT.md +0 -20
- package/prompts/languages/BASH.md +0 -22
- package/prompts/languages/C.md +0 -21
- package/prompts/languages/CSHARP.md +0 -24
- package/prompts/languages/CSS_SCSS.md +0 -22
- package/prompts/languages/GO.md +0 -24
- package/prompts/languages/JAVA.md +0 -23
- package/prompts/languages/JAVASCRIPT_TYPESCRIPT.md +0 -24
- package/prompts/languages/KOTLIN.md +0 -24
- package/prompts/languages/PHP.md +0 -23
- package/prompts/languages/PYTHON.md +0 -23
- package/prompts/languages/RUBY.md +0 -23
- package/prompts/languages/RUST.md +0 -21
- package/prompts/languages/SWIFT.md +0 -21
- package/seal.mjs +0 -400
- package/search-core.mjs +0 -777
- package/search-sparse.mjs +0 -183
- package/storage.mjs +0 -73
- package/watch-daemon.mjs +0 -289
package/config.mjs
DELETED
|
@@ -1,494 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file config.mjs
|
|
3
|
-
* @description Single source of truth for runtime configuration. Resolves, in
|
|
4
|
-
* order of precedence, CLI flags > environment variables >
|
|
5
|
-
* `.graph-indexer.json` > built-in defaults, into one frozen object
|
|
6
|
-
* consumed by the indexer, watch daemon and MCP server.
|
|
7
|
-
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
8
|
-
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
9
|
-
* @license MIT
|
|
10
|
-
*/
|
|
11
|
-
import fs from 'fs';
|
|
12
|
-
import path from 'path';
|
|
13
|
-
import { artifactPaths, CONFIG_FILE_NAME, DATA_DIR_NAME } from './layout.mjs';
|
|
14
|
-
import { assertSealCompatible } from './seal.mjs';
|
|
15
|
-
|
|
16
|
-
export const DEFAULTS = Object.freeze({
|
|
17
|
-
// 'auto' keeps the index in memory below AUTO_SQLITE_CHUNK_THRESHOLD chunks and
|
|
18
|
-
// switches to disk-backed SQLite above it. Force either with --use-sqlite /
|
|
19
|
-
// INDEXER_STORAGE. (see storage.mjs: resolveBackend / AUTO_SQLITE_CHUNK_THRESHOLD)
|
|
20
|
-
storage: 'auto', // 'auto' (default) | 'memory' | 'sqlite'
|
|
21
|
-
embeddings: false, // OFF by default — lexical + stemming need zero dependencies.
|
|
22
|
-
// Opt in with --embeddings or INDEXER_EMBEDDINGS=on.
|
|
23
|
-
embedProvider: 'auto',
|
|
24
|
-
// 'auto' → Ollama (if running w/ the model) → in-process local → lexical-only
|
|
25
|
-
// 'ollama' → force Ollama daemon (requires: ollama serve + pulled model)
|
|
26
|
-
// 'local' → in-process Xenova MiniLM (requires: npm i @huggingface/transformers)
|
|
27
|
-
// 'code-local'→ in-process code-specialized embedder (jina-code; same optional dep as
|
|
28
|
-
// 'local'; ~160 MB q8 first download, air-gapped after) — sharper code recall
|
|
29
|
-
// 'mlx' → Apple Metal GPU (requires: npm run embed:setup:mlx; macOS only)
|
|
30
|
-
// 'off' → lexical-only, no vectors
|
|
31
|
-
embedModel: 'nomic-embed-text', // default Ollama embed model when embeddings are enabled.
|
|
32
|
-
// qwen3-embedding:4b is a documented opt-in upgrade
|
|
33
|
-
// (--embed-model / EMBED_MODEL), never a hardcoded default.
|
|
34
|
-
localEmbedModel: 'Xenova/all-MiniLM-L6-v2', // in-process model (optional @huggingface/transformers)
|
|
35
|
-
codeEmbedModel: 'jinaai/jina-embeddings-v2-base-code', // in-process CODE-specialized model for
|
|
36
|
-
// provider 'code-local' (768-dim, MIT, 8192-token context).
|
|
37
|
-
// Override with --code-embed-model / INDEXER_CODE_EMBED_MODEL.
|
|
38
|
-
mlxEmbedModel: 'mlx-community/all-MiniLM-L6-v2-4bit', // MLX embedder model (provider 'mlx').
|
|
39
|
-
// Override with --mlx-embed-model / INDEXER_MLX_EMBED_MODEL;
|
|
40
|
-
// must be an mlx_embeddings-compatible sentence model.
|
|
41
|
-
ollamaHost: 'http://localhost:11434',
|
|
42
|
-
llmProvider: 'ollama', // LLM backend for enrichment, reranking, HyDE: 'ollama' | 'mlx'
|
|
43
|
-
// 'mlx' requires mlx_lm.server running (mlx_lm.server --model <name>)
|
|
44
|
-
mlxLmHost: 'http://localhost:8080', // mlx_lm.server endpoint (default port 8080)
|
|
45
|
-
gitSignals: true, // collect local git churn/recency/co-change at index time (air-gapped)
|
|
46
|
-
gitRankBoost: 0, // 0..1 opt-in recency/churn weight in search_code (0 = ranking unchanged)
|
|
47
|
-
interprocedural: false, // OPT-IN: index-time inter-procedural receiver-type fixpoint —
|
|
48
|
-
// propagate return types along factory call chains so multi-hop
|
|
49
|
-
// receivers resolve in get_call_graph / find_references. Air-gapped,
|
|
50
|
-
// deterministic; default index is byte-identical. Enable with
|
|
51
|
-
// --interprocedural / INDEXER_INTERPROCEDURAL.
|
|
52
|
-
symbolGraph: false, // OPT-IN: persist a resolved symbol graph (chunk→chunk edges with
|
|
53
|
-
// kind + confidence) at index time, exposed via getEdges. findCallers/
|
|
54
|
-
// findReferers read it when present (identical sets, fall back to scan).
|
|
55
|
-
// Air-gapped, deterministic; default index byte-identical; search
|
|
56
|
-
// ranking unaffected. Enable with --symbol-graph / INDEXER_SYMBOL_GRAPH.
|
|
57
|
-
taint: false, // OPT-IN: precompute + serialize taint flows (C2) at index time so the
|
|
58
|
-
// trace_taint / find_tainted_sinks tools serve instantly (both backends
|
|
59
|
-
// serve identical flows). Default off → tools compute query-time (the
|
|
60
|
-
// index is byte-identical). --taint / INDEXER_TAINT.
|
|
61
|
-
ranker: 'rrf', // Search ranker (D3). 'rrf' (default): the RRF + boost-ladder fusion —
|
|
62
|
-
// BYTE-IDENTICAL to before. 'learned': a post-fusion linear re-rank over
|
|
63
|
-
// RRF score + A5 centrality + resolved/SCIP in-degree + git (zero-dep dot
|
|
64
|
-
// product; lift is on --symbol-graph indexes). --ranker / INDEXER_RANKER.
|
|
65
|
-
learnedSparse: false, // OPT-IN: learn a corpus co-occurrence (PMI) association model (B3) at
|
|
66
|
-
// index time and serialize it, adding an NL-only vocabulary-EXPANSION
|
|
67
|
-
// channel at query time (zero new dep, fully air-gapped). Default off →
|
|
68
|
-
// no model → byte-identical. --learned-sparse / INDEXER_LEARNED_SPARSE.
|
|
69
|
-
sealed: 'off', // Sealed mode (F1): 'off' (default) | 'local' (loopback only) |
|
|
70
|
-
// 'strict' (zero network egress; lexical-only). Fail-closed:
|
|
71
|
-
// refuses to start if an enabled feature would egress beyond the
|
|
72
|
-
// tier, and installs a deny-by-default runtime egress guard.
|
|
73
|
-
// --sealed [strict|local] / INDEXER_SEALED.
|
|
74
|
-
resolver: 'heuristic', // Resolver provider for the symbol graph (A1/A2). 'heuristic' (default):
|
|
75
|
-
// edges stay { high, name_only } — byte-identical. 'precise' (A1): lift
|
|
76
|
-
// provably-unambiguous edges (sole definition / type-pinned receiver)
|
|
77
|
-
// to a `resolved` tier, powering impact_of_edit precision='strict'.
|
|
78
|
-
// 'scip' (A2): ingest a locally-generated SCIP index (--scip-index) and
|
|
79
|
-
// promote/suppress `calls` edges by real cross-file bindings. Only
|
|
80
|
-
// meaningful with --symbol-graph. --resolver / INDEXER_RESOLVER.
|
|
81
|
-
scipIndex: null, // Path to a locally-generated SCIP index for --resolver scip (A2).
|
|
82
|
-
// --scip-index / INDEXER_SCIP_INDEX / scipIndex config; null = none.
|
|
83
|
-
enrichment: Object.freeze({
|
|
84
|
-
enabled: false,
|
|
85
|
-
model: 'qwen2.5-coder:1.5b', // smallest coder model; quality sufficient for summaries.
|
|
86
|
-
// Opt in with --enrichment or ENRICH_MODEL=<model>.
|
|
87
|
-
coreRatio: 1.0, // 1.0 = all production files (tests/examples always excluded);
|
|
88
|
-
// <1 bounds enrichment to the most-central share by PageRank
|
|
89
|
-
maxChunks: 500, // cap on NEW LLM calls per index run (cache accumulates across runs)
|
|
90
|
-
concurrency: 4, // parallel Ollama requests during enrichment; keep low —
|
|
91
|
-
// a single local model serves requests fastest one-at-a-time
|
|
92
|
-
// unless OLLAMA_NUM_PARALLEL + hardware allow more
|
|
93
|
-
|
|
94
|
-
}),
|
|
95
|
-
rerank: Object.freeze({
|
|
96
|
-
enabled: false, // opt-in: one rerank pass per natural-language query
|
|
97
|
-
provider: 'generative', // 'generative' = LLM judge (default) | 'cross-encoder' = local
|
|
98
|
-
// air-gapped MS-MARCO cross-encoder (no Ollama/daemon) |
|
|
99
|
-
// 'late-interaction' = ColBERT-style multi-vector MaxSim (B4,
|
|
100
|
-
// reuses the embedder + stored doc vectors; needs --embeddings).
|
|
101
|
-
// Set with --rerank-provider / RERANK_PROVIDER.
|
|
102
|
-
model: 'qwen2.5-coder:7b', // generative judge model: 7B measured +50% semantic rank-1, 1.5B ~nil
|
|
103
|
-
crossEncoderModel: 'Xenova/ms-marco-MiniLM-L-6-v2', // cross-encoder provider model
|
|
104
|
-
// (optional @huggingface/transformers; downloaded on first use)
|
|
105
|
-
topM: 12, // candidates the reranker reorders
|
|
106
|
-
poolSize: 15, // over-fetch depth when reranking, so a correct-but-deep
|
|
107
|
-
// hit can be RESCUED into top_k (then truncated to top_k)
|
|
108
|
-
}),
|
|
109
|
-
hyde: Object.freeze({
|
|
110
|
-
enabled: false, // opt-in: query-side HyDE — one LLM call per NL query, blended into the query vector
|
|
111
|
-
model: 'qwen2.5-coder:1.5b', // local coder model that writes the hypothetical snippet
|
|
112
|
-
}),
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
/**
|
|
116
|
-
* Reads the project config, tolerating absence/corruption. Looks first inside the
|
|
117
|
-
* tidy data dir (`.graph-indexer/config.json`), then falls back to the legacy
|
|
118
|
-
* root file (`.graph-indexer.json`) so pre-v1.4 projects keep working until they
|
|
119
|
-
* are migrated.
|
|
120
|
-
*/
|
|
121
|
-
export function loadConfigFile(root) {
|
|
122
|
-
const candidates = [
|
|
123
|
-
path.join(root, DATA_DIR_NAME, CONFIG_FILE_NAME),
|
|
124
|
-
path.join(root, '.graph-indexer.json'), // legacy root config (back-compat)
|
|
125
|
-
];
|
|
126
|
-
for (const configPath of candidates) {
|
|
127
|
-
try {
|
|
128
|
-
if (fs.existsSync(configPath)) return JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
|
129
|
-
} catch { /* malformed config — try the next candidate, else defaults */ }
|
|
130
|
-
}
|
|
131
|
-
return {};
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
/** Returns the value following `--flag` in argv, or undefined. */
|
|
135
|
-
function flagValue(argv, flag) {
|
|
136
|
-
const i = argv.indexOf(flag);
|
|
137
|
-
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : undefined;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Resolve the effective configuration.
|
|
142
|
-
*
|
|
143
|
-
* @param {object} [opts]
|
|
144
|
-
* @param {string[]} [opts.argv] Defaults to process.argv.slice(2).
|
|
145
|
-
* @param {object} [opts.env] Defaults to process.env.
|
|
146
|
-
* @param {string} [opts.cwd] Defaults to process.cwd().
|
|
147
|
-
* @returns {Readonly<object>}
|
|
148
|
-
*/
|
|
149
|
-
export function resolveConfig({ argv = process.argv.slice(2), env = process.env, cwd = process.cwd() } = {}) {
|
|
150
|
-
// Project root: --repo wins, then MCP_PROJECT_ROOT, then cwd. The config file,
|
|
151
|
-
// index artifacts and language selection are all anchored to this directory.
|
|
152
|
-
const repoArg = flagValue(argv, '--repo');
|
|
153
|
-
const projectRoot = path.resolve(repoArg || env.MCP_PROJECT_ROOT || cwd);
|
|
154
|
-
|
|
155
|
-
const file = loadConfigFile(projectRoot);
|
|
156
|
-
const fileEnrich = file.enrichment || {};
|
|
157
|
-
|
|
158
|
-
// Storage: --use-sqlite forces SQLite; otherwise INDEXER_STORAGE / "storage" key
|
|
159
|
-
// / default 'auto' (resolved to memory-or-sqlite by chunk count at index time and
|
|
160
|
-
// by artifact presence at query time — see storage.mjs:resolveBackend).
|
|
161
|
-
const storageRaw = argv.includes('--use-sqlite') ? 'sqlite'
|
|
162
|
-
: (env.INDEXER_STORAGE || file.storage || DEFAULTS.storage);
|
|
163
|
-
const storage = ['memory', 'sqlite', 'auto'].includes(storageRaw) ? storageRaw : DEFAULTS.storage;
|
|
164
|
-
|
|
165
|
-
// Enrichment is OFF unless explicitly requested: --enrichment flag, ENRICH_MODEL
|
|
166
|
-
// env (naming a model implies enabling it), or the project config.
|
|
167
|
-
const enrichModelEnv = env.ENRICH_MODEL || undefined;
|
|
168
|
-
const enrichmentEnabled = argv.includes('--enrichment')
|
|
169
|
-
|| Boolean(enrichModelEnv)
|
|
170
|
-
|| Boolean(fileEnrich.enabled);
|
|
171
|
-
|
|
172
|
-
const ollamaHost = env.OLLAMA_HOST || file.ollamaHost || DEFAULTS.ollamaHost;
|
|
173
|
-
|
|
174
|
-
// LLM provider for enrichment / reranking / HyDE: env > flag > config > default.
|
|
175
|
-
// 'mlx' routes generation through a local mlx_lm.server (OpenAI-compatible).
|
|
176
|
-
const llmProviderRaw = env.INDEXER_LLM_PROVIDER || flagValue(argv, '--llm-provider') || file.llmProvider || DEFAULTS.llmProvider;
|
|
177
|
-
const llmProvider = ['ollama', 'mlx'].includes(llmProviderRaw) ? llmProviderRaw : DEFAULTS.llmProvider;
|
|
178
|
-
const mlxLmHost = env.INDEXER_MLX_LM_HOST || flagValue(argv, '--mlx-lm-host') || file.mlxLmHost || DEFAULTS.mlxLmHost;
|
|
179
|
-
|
|
180
|
-
// Embeddings are OFF by default (lexical + stemming need zero dependencies).
|
|
181
|
-
// Enable with --embeddings or INDEXER_EMBEDDINGS=on; INDEXER_EMBEDDINGS=off always wins.
|
|
182
|
-
const embeddingsEnabled = env.INDEXER_EMBEDDINGS === 'off'
|
|
183
|
-
? false
|
|
184
|
-
: (argv.includes('--embeddings') || env.INDEXER_EMBEDDINGS === 'on' || file.embeddings === true);
|
|
185
|
-
|
|
186
|
-
// Embedding provider: env > --embed-provider flag > config > default 'auto'.
|
|
187
|
-
// 'auto' prefers a running Ollama and falls back to the in-process local model
|
|
188
|
-
// (optional @huggingface/transformers) so semantic search works with no daemon.
|
|
189
|
-
const embedProvider = env.INDEXER_EMBED_PROVIDER
|
|
190
|
-
|| flagValue(argv, '--embed-provider')
|
|
191
|
-
|| file.embedProvider || DEFAULTS.embedProvider;
|
|
192
|
-
|
|
193
|
-
// Git signals: collection is air-gapped (local `git log` only). Disable with
|
|
194
|
-
// INDEXER_GIT_SIGNALS=off, --no-git-signals, or "gitSignals": false.
|
|
195
|
-
const gitSignals = env.INDEXER_GIT_SIGNALS === 'off' || argv.includes('--no-git-signals')
|
|
196
|
-
? false
|
|
197
|
-
: (file.gitSignals === false ? false : DEFAULTS.gitSignals);
|
|
198
|
-
const gitRankBoostRaw = env.INDEXER_GIT_RANK_BOOST ?? flagValue(argv, '--git-rank-boost')
|
|
199
|
-
?? (Number.isFinite(file.gitRankBoost) ? file.gitRankBoost : DEFAULTS.gitRankBoost);
|
|
200
|
-
const gitRankBoost = Math.min(1, Math.max(0, Number(gitRankBoostRaw) || 0));
|
|
201
|
-
|
|
202
|
-
// Opt-in inter-procedural receiver-type fixpoint (index-time, air-gapped).
|
|
203
|
-
const interprocedural = argv.includes('--interprocedural')
|
|
204
|
-
|| env.INDEXER_INTERPROCEDURAL === 'on'
|
|
205
|
-
|| file.interprocedural === true;
|
|
206
|
-
|
|
207
|
-
// Opt-in persistent resolved symbol graph (index-time, air-gapped).
|
|
208
|
-
const symbolGraph = argv.includes('--symbol-graph')
|
|
209
|
-
|| env.INDEXER_SYMBOL_GRAPH === 'on'
|
|
210
|
-
|| file.symbolGraph === true;
|
|
211
|
-
|
|
212
|
-
// Opt-in index-time taint serialization (C2): precompute + persist the flow set.
|
|
213
|
-
const taint = argv.includes('--taint')
|
|
214
|
-
|| env.INDEXER_TAINT === 'on'
|
|
215
|
-
|| file.taint === true;
|
|
216
|
-
|
|
217
|
-
// Opt-in learned-sparse model (B3): learn + persist the corpus association model at index time.
|
|
218
|
-
const learnedSparse = argv.includes('--learned-sparse')
|
|
219
|
-
|| env.INDEXER_LEARNED_SPARSE === 'on'
|
|
220
|
-
|| file.learnedSparse === true;
|
|
221
|
-
|
|
222
|
-
// Search ranker (D3). Unknown values fall back to the default 'rrf' (byte-identical fusion).
|
|
223
|
-
const rankerRaw = flagValue(argv, '--ranker') || env.INDEXER_RANKER || file.ranker || DEFAULTS.ranker;
|
|
224
|
-
const ranker = (rankerRaw === 'learned' || rankerRaw === 'rrf') ? rankerRaw : 'rrf';
|
|
225
|
-
|
|
226
|
-
// Resolver provider for the symbol graph (A1/A2). Unknown values fall back to 'heuristic'.
|
|
227
|
-
const resolverRaw = flagValue(argv, '--resolver') || env.INDEXER_RESOLVER || file.resolver || DEFAULTS.resolver;
|
|
228
|
-
const resolver = (resolverRaw === 'precise' || resolverRaw === 'heuristic' || resolverRaw === 'scip')
|
|
229
|
-
? resolverRaw : 'heuristic';
|
|
230
|
-
// A2: path to a locally-generated SCIP index, consumed by the `scip` resolver. flag > env > file.
|
|
231
|
-
// Resolved against projectRoot (a relative path) so a config-file value is repo-relative.
|
|
232
|
-
const scipRaw = flagValue(argv, '--scip-index') || env.INDEXER_SCIP_INDEX || file.scipIndex || null;
|
|
233
|
-
const scipIndex = scipRaw ? path.resolve(projectRoot, scipRaw) : null;
|
|
234
|
-
|
|
235
|
-
// Sealed mode (F1). flag > env > file > default. A bare `--sealed` (no value, or followed by
|
|
236
|
-
// another flag) means strict; `--sealed local` opts into the loopback tier; env 'on' → strict.
|
|
237
|
-
const sealedFlagPresent = argv.includes('--sealed');
|
|
238
|
-
const sealedFlagVal = sealedFlagPresent ? flagValue(argv, '--sealed') : undefined;
|
|
239
|
-
let sealedRaw;
|
|
240
|
-
if (sealedFlagPresent) sealedRaw = (sealedFlagVal && !sealedFlagVal.startsWith('--')) ? sealedFlagVal : 'strict';
|
|
241
|
-
else if (env.INDEXER_SEALED) sealedRaw = env.INDEXER_SEALED;
|
|
242
|
-
else sealedRaw = typeof file.sealed === 'string' ? file.sealed : DEFAULTS.sealed;
|
|
243
|
-
const sealed = sealedRaw === 'on' ? 'strict'
|
|
244
|
-
: (['strict', 'local', 'off'].includes(sealedRaw) ? sealedRaw : 'off');
|
|
245
|
-
|
|
246
|
-
const paths = artifactPaths(projectRoot);
|
|
247
|
-
|
|
248
|
-
const resolved = Object.freeze({
|
|
249
|
-
projectRoot,
|
|
250
|
-
sealed,
|
|
251
|
-
storage,
|
|
252
|
-
dataDir: paths.dataDir,
|
|
253
|
-
indexPath: paths.indexPath,
|
|
254
|
-
embeddingPath: paths.embeddingPath,
|
|
255
|
-
sqlitePath: paths.sqlitePath,
|
|
256
|
-
enrichmentCachePath: paths.enrichmentCachePath,
|
|
257
|
-
gitSignalsPath: paths.gitSignalsPath,
|
|
258
|
-
pidFile: paths.pidFile,
|
|
259
|
-
logFile: paths.logFile,
|
|
260
|
-
|
|
261
|
-
languages: Array.isArray(file.languages) ? file.languages : null, // null = all
|
|
262
|
-
|
|
263
|
-
ollamaHost,
|
|
264
|
-
llmProvider,
|
|
265
|
-
mlxLmHost,
|
|
266
|
-
embeddingsEnabled,
|
|
267
|
-
embedProvider,
|
|
268
|
-
// Embed model: --embed-model flag > EMBED_MODEL env > config > nomic default.
|
|
269
|
-
// This is the model `auto`/`ollama` providers pull (e.g. qwen3-embedding:4b).
|
|
270
|
-
embedModel: flagValue(argv, '--embed-model') || env.EMBED_MODEL || file.embedModel || DEFAULTS.embedModel,
|
|
271
|
-
localEmbedModel: file.localEmbedModel || DEFAULTS.localEmbedModel,
|
|
272
|
-
// Code-specialized in-process model (provider 'code-local'): --code-embed-model flag >
|
|
273
|
-
// INDEXER_CODE_EMBED_MODEL env > config > jina-code default. Stamped into the embed-meta
|
|
274
|
-
// so query time and index time agree, and a model switch forces a clean re-embed.
|
|
275
|
-
codeEmbedModel: flagValue(argv, '--code-embed-model') || env.INDEXER_CODE_EMBED_MODEL
|
|
276
|
-
|| file.codeEmbedModel || DEFAULTS.codeEmbedModel,
|
|
277
|
-
// MLX embedder model: --mlx-embed-model flag > INDEXER_MLX_EMBED_MODEL env >
|
|
278
|
-
// config > pinned MiniLM default. Passed to the Python server + stamped in meta.
|
|
279
|
-
mlxEmbedModel: flagValue(argv, '--mlx-embed-model') || env.INDEXER_MLX_EMBED_MODEL
|
|
280
|
-
|| file.mlxEmbedModel || DEFAULTS.mlxEmbedModel,
|
|
281
|
-
gitSignals,
|
|
282
|
-
gitRankBoost,
|
|
283
|
-
interprocedural,
|
|
284
|
-
symbolGraph,
|
|
285
|
-
taint,
|
|
286
|
-
ranker,
|
|
287
|
-
resolver,
|
|
288
|
-
scipIndex,
|
|
289
|
-
learnedSparse,
|
|
290
|
-
|
|
291
|
-
enrichment: Object.freeze({
|
|
292
|
-
enabled: enrichmentEnabled,
|
|
293
|
-
model: flagValue(argv, '--enrich-model') || enrichModelEnv || fileEnrich.model || DEFAULTS.enrichment.model,
|
|
294
|
-
coreRatio: Number(fileEnrich.coreRatio) > 0 ? Number(fileEnrich.coreRatio) : DEFAULTS.enrichment.coreRatio,
|
|
295
|
-
maxChunks: Number(flagValue(argv, '--enrich-max')) > 0
|
|
296
|
-
? Number(flagValue(argv, '--enrich-max'))
|
|
297
|
-
: (Number.isInteger(fileEnrich.maxChunks) ? fileEnrich.maxChunks : DEFAULTS.enrichment.maxChunks),
|
|
298
|
-
concurrency: Number(flagValue(argv, '--enrich-concurrency')) > 0
|
|
299
|
-
? Number(flagValue(argv, '--enrich-concurrency'))
|
|
300
|
-
: (Number(fileEnrich.concurrency) > 0 ? Number(fileEnrich.concurrency) : DEFAULTS.enrichment.concurrency),
|
|
301
|
-
}),
|
|
302
|
-
|
|
303
|
-
rerank: (() => {
|
|
304
|
-
// Provider: env > --rerank-provider flag > config; validated to a known provider.
|
|
305
|
-
const provRaw = env.RERANK_PROVIDER || flagValue(argv, '--rerank-provider') || (file.rerank || {}).provider;
|
|
306
|
-
const provider = ['generative', 'cross-encoder', 'late-interaction'].includes(provRaw) ? provRaw : DEFAULTS.rerank.provider;
|
|
307
|
-
return Object.freeze({
|
|
308
|
-
// OFF unless requested: --rerank flag, RERANK_MODEL env (naming a model implies
|
|
309
|
-
// enabling it), an explicit provider (env/flag), or the project config.
|
|
310
|
-
enabled: argv.includes('--rerank') || Boolean(env.RERANK_MODEL) || Boolean(provRaw)
|
|
311
|
-
|| Boolean((file.rerank || {}).enabled),
|
|
312
|
-
provider,
|
|
313
|
-
model: env.RERANK_MODEL || (file.rerank || {}).model || DEFAULTS.rerank.model,
|
|
314
|
-
crossEncoderModel: env.RERANK_CROSS_ENCODER_MODEL || (file.rerank || {}).crossEncoderModel
|
|
315
|
-
|| DEFAULTS.rerank.crossEncoderModel,
|
|
316
|
-
topM: Number.isInteger((file.rerank || {}).topM) ? (file.rerank || {}).topM : DEFAULTS.rerank.topM,
|
|
317
|
-
poolSize: Number.isInteger((file.rerank || {}).poolSize) ? (file.rerank || {}).poolSize : DEFAULTS.rerank.poolSize,
|
|
318
|
-
});
|
|
319
|
-
})(),
|
|
320
|
-
|
|
321
|
-
hyde: Object.freeze({
|
|
322
|
-
enabled: Boolean((file.hyde || {}).enabled),
|
|
323
|
-
model: (file.hyde || {}).model || DEFAULTS.hyde.model,
|
|
324
|
-
}),
|
|
325
|
-
});
|
|
326
|
-
|
|
327
|
-
// F1: fail-closed. If --sealed was requested and an enabled feature would egress beyond the
|
|
328
|
-
// tier, throw here so a sealed-incompatible config can never be constructed (no-op when off).
|
|
329
|
-
assertSealCompatible(resolved);
|
|
330
|
-
return resolved;
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
// Memoised singleton for import-time consumers (e.g. parse/languages.mjs language loading).
|
|
334
|
-
let _cached = null;
|
|
335
|
-
export function getConfig() {
|
|
336
|
-
if (!_cached) _cached = resolveConfig();
|
|
337
|
-
return _cached;
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
/**
|
|
341
|
-
* Human-readable lines describing the effective configuration, printed at startup
|
|
342
|
-
* by the indexer / MCP server / daemon so users can see exactly what is running
|
|
343
|
-
* (storage backend, model names, which optional features are on).
|
|
344
|
-
*
|
|
345
|
-
* @param {object} config Resolved config from resolveConfig().
|
|
346
|
-
* @param {object} [opts]
|
|
347
|
-
* @param {string} [opts.backend] Concrete backend ('memory'|'sqlite') once 'auto'
|
|
348
|
-
* has been resolved; defaults to config.storage.
|
|
349
|
-
* @returns {string[]}
|
|
350
|
-
*/
|
|
351
|
-
export function describeConfig(config, { backend = config.storage } = {}) {
|
|
352
|
-
// Show the model that the chosen provider actually loads (mlx/local/code-local use their own).
|
|
353
|
-
const provModel = config.embedProvider === 'mlx' ? config.mlxEmbedModel
|
|
354
|
-
: config.embedProvider === 'local' ? config.localEmbedModel
|
|
355
|
-
: config.embedProvider === 'code-local' ? config.codeEmbedModel
|
|
356
|
-
: config.embedModel;
|
|
357
|
-
const emb = config.embeddingsEnabled
|
|
358
|
-
? `on · provider=${config.embedProvider}, model=${provModel}`
|
|
359
|
-
: 'off (lexical + stemming only)';
|
|
360
|
-
return [
|
|
361
|
-
`storage : ${backend}`,
|
|
362
|
-
`embeddings : ${emb}`,
|
|
363
|
-
`enrichment : ${config.enrichment.enabled ? `on · model=${config.enrichment.model} · provider=${config.llmProvider}` : 'off'}`,
|
|
364
|
-
`reranker : ${config.rerank.enabled
|
|
365
|
-
? (config.rerank.provider === 'cross-encoder'
|
|
366
|
-
? `on · provider=cross-encoder · model=${config.rerank.crossEncoderModel}`
|
|
367
|
-
: config.rerank.provider === 'late-interaction'
|
|
368
|
-
? 'on · provider=late-interaction · ColBERT-style multi-vector MaxSim (needs --embeddings)'
|
|
369
|
-
: `on · provider=generative · model=${config.rerank.model} · llm=${config.llmProvider}`)
|
|
370
|
-
: 'off'}`,
|
|
371
|
-
`git signals : ${config.gitSignals ? 'on' : 'off'}${config.gitRankBoost ? ` · rank boost=${config.gitRankBoost}` : ''}`,
|
|
372
|
-
`interproc. : ${config.interprocedural ? 'on · factory return-type propagation' : 'off'}`,
|
|
373
|
-
`symbol graph: ${config.symbolGraph ? 'on · persisted resolved edges (getEdges)' : 'off'}`,
|
|
374
|
-
`taint : ${config.taint ? 'on · precomputed flows (trace_taint / find_tainted_sinks)' : 'off (query-time)'}`,
|
|
375
|
-
`ranker : ${config.ranker === 'learned' ? 'learned · post-fusion linear re-rank (opt-in)' : 'rrf (default)'}`,
|
|
376
|
-
`sparse : ${config.learnedSparse ? 'on · learned-sparse vocabulary expansion (NL-only channel)' : 'off'}`,
|
|
377
|
-
`resolver : ${config.resolver === 'precise' ? 'precise · unambiguous edges → `resolved` tier'
|
|
378
|
-
: config.resolver === 'scip' ? `scip · cross-file SCIP bindings → \`resolved\` (${config.scipIndex || '⚠️ no --scip-index'})`
|
|
379
|
-
: 'heuristic (default)'}`,
|
|
380
|
-
`sealed mode : ${config.sealed === 'strict' ? '🔒 strict · zero network egress (enforced)'
|
|
381
|
-
: config.sealed === 'local' ? '🔒 local · loopback only (enforced)' : 'off'}`,
|
|
382
|
-
];
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
/**
|
|
386
|
-
* Operational warnings for risky / edge configurations. The transition between
|
|
387
|
-
* default and an opt-in feature must never be silent: callers print these at
|
|
388
|
-
* startup so a user who enabled enrichment-only or the reranker sees the measured
|
|
389
|
-
* trade-off before they wonder why precision moved.
|
|
390
|
-
*
|
|
391
|
-
* @param {object} config Resolved config from resolveConfig().
|
|
392
|
-
* @returns {string[]}
|
|
393
|
-
*/
|
|
394
|
-
export function configNotices(config) {
|
|
395
|
-
const out = [];
|
|
396
|
-
if (config.sealed === 'strict' || config.sealed === 'local') {
|
|
397
|
-
out.push(`Sealed mode (${config.sealed}): a deny-by-default egress guard is installed — `
|
|
398
|
-
+ `${config.sealed === 'strict' ? 'NO network connections are permitted' : 'only loopback connections are permitted'}. `
|
|
399
|
-
+ 'The build refused to start if any enabled feature would egress beyond the tier. '
|
|
400
|
-
+ 'The in-process guard (sockets/http/fetch) is airtight; the child_process denylist is '
|
|
401
|
-
+ 'best-effort. Run `idx-index --attest` for the signed-able manifest.');
|
|
402
|
-
}
|
|
403
|
-
if (config.embeddingsEnabled && config.embedProvider === 'code-local') {
|
|
404
|
-
out.push(`Code-specialized embedder (code-local): the dense channel is built in-process by `
|
|
405
|
-
+ `${config.codeEmbedModel} — a CODE-trained model (identifiers, call patterns, type names) `
|
|
406
|
-
+ 'that sharpens semantic recall on typed languages where the general-purpose MiniLM is '
|
|
407
|
-
+ 'weakest. It needs the optional \'@huggingface/transformers\' package; the quantized '
|
|
408
|
-
+ '(q8) model is ~160 MB on the FIRST run (downloaded once, then fully air-gapped — no daemon, no network). '
|
|
409
|
-
+ 'It is slower per chunk than MiniLM and produces 768-dim vectors — switching to it from '
|
|
410
|
-
+ 'another provider forces a clean re-embed (vectors of different models must not be mixed). '
|
|
411
|
-
+ 'Pair with the persistent embed cache to pay the embedding cost only once. Measure on '
|
|
412
|
-
+ 'YOUR repo (`npm run test:eval -- --embeddings --embed-provider code-local`) before adopting.');
|
|
413
|
-
}
|
|
414
|
-
if (config.interprocedural) {
|
|
415
|
-
out.push('Inter-procedural receiver types are resolved at index time (a whole-program pass). '
|
|
416
|
-
+ 'The watch daemon updates one file at a time and does NOT re-run the fixpoint, so '
|
|
417
|
-
+ 'cross-file factory chains refresh on the next full `idx-index`; until then they fall '
|
|
418
|
-
+ 'back to single-hop resolution (never worse than the default). It only sharpens '
|
|
419
|
-
+ 'get_call_graph / find_references — search ranking is unaffected.');
|
|
420
|
-
}
|
|
421
|
-
if (config.symbolGraph) {
|
|
422
|
-
out.push('The resolved symbol graph is built once at index time (a whole-program pass), adding '
|
|
423
|
-
+ 'index time roughly proportional to repo size. The watch daemon does NOT rebuild it, so '
|
|
424
|
-
+ 'edges refresh on the next full `idx-index`; until then findCallers/findReferers fall back '
|
|
425
|
-
+ 'to the name-match scan (never worse). It powers getEdges; search ranking is unaffected.');
|
|
426
|
-
}
|
|
427
|
-
if (config.ranker === 'learned') {
|
|
428
|
-
out.push('Learned ranker (D3): the fused top-N is re-ordered by a zero-dependency linear model '
|
|
429
|
-
+ 'over the RRF score + symbol centrality + resolved/SCIP in-degree + git recency. It is a '
|
|
430
|
-
+ 'post-fusion re-rank (the default RRF ordering is untouched). Its discriminating features '
|
|
431
|
-
+ 'are zero without --symbol-graph, so pair it with --symbol-graph for the measured lift; on a '
|
|
432
|
-
+ 'plain lexical index it is ≈neutral. RRF remains the default; this is opt-in.');
|
|
433
|
-
}
|
|
434
|
-
if (config.taint) {
|
|
435
|
-
out.push('Taint flows are precomputed once at index time and serialized, so trace_taint / '
|
|
436
|
-
+ 'find_tainted_sinks answer instantly (both backends serve identical flows). The watch '
|
|
437
|
-
+ 'daemon does NOT rebuild them — flows refresh on the next full `idx-index`; until then the '
|
|
438
|
-
+ 'tools fall back to a query-time scan (never worse). It only affects the security tools; '
|
|
439
|
-
+ 'search ranking is unaffected. Pair with --symbol-graph for cross-function precision.');
|
|
440
|
-
}
|
|
441
|
-
if (config.learnedSparse) {
|
|
442
|
-
out.push('Learned-sparse (B3): a corpus co-occurrence (PMI) association model is learned + '
|
|
443
|
-
+ 'serialized at index time, adding a vocabulary-EXPANSION channel that fires ONLY for '
|
|
444
|
-
+ 'natural-language queries (symbolic/exact lookups stay byte-identical). Zero new dependency, '
|
|
445
|
-
+ 'fully air-gapped, parity-safe. The whole-program model refreshes on the next full '
|
|
446
|
-
+ '`idx-index` (the daemon drops it on a per-file edit). Measure on YOUR repo (`test:eval`) — '
|
|
447
|
-
+ 'it is a recall aid that helps lexical-gap semantic queries and is ≈neutral elsewhere.');
|
|
448
|
-
}
|
|
449
|
-
if ((config.resolver === 'precise' || config.resolver === 'scip') && !config.symbolGraph) {
|
|
450
|
-
out.push(`--resolver ${config.resolver} has no effect without --symbol-graph (the resolver runs only `
|
|
451
|
-
+ 'when the symbol graph is built). Add --symbol-graph to enable the `resolved` edge tier.');
|
|
452
|
-
} else if (config.resolver === 'precise') {
|
|
453
|
-
out.push('Precise resolver: edges with a provably-unambiguous binding (sole definition, or a '
|
|
454
|
-
+ 'type-pinned receiver) are promoted from `high` to a `resolved` tier — used by '
|
|
455
|
-
+ "impact_of_edit(precision: 'strict') for a false-positive-free blast radius. Confidence "
|
|
456
|
-
+ 'strings only change; edge sets, parity, and search ranking are unaffected.');
|
|
457
|
-
} else if (config.resolver === 'scip' && !config.scipIndex) {
|
|
458
|
-
out.push('--resolver scip needs --scip-index <path.scip> (a SCIP index generated locally by your '
|
|
459
|
-
+ 'own toolchain — scip-typescript, scip-python, scip-java, rust-analyzer --scip). Without it '
|
|
460
|
-
+ 'the resolver is inert and edges stay heuristic { high, name_only }.');
|
|
461
|
-
} else if (config.resolver === 'scip') {
|
|
462
|
-
out.push('SCIP resolver: cross-file bindings from a locally-generated SCIP index promote CONFIRMED '
|
|
463
|
-
+ '`calls` edges to a `resolved` tier AND suppress the wrong-target fan-out an ambiguous name '
|
|
464
|
-
+ 'would otherwise emit (heritage/type edges stay heuristic). Suppression is sound under '
|
|
465
|
-
+ 'partial coverage — a def SCIP never recorded is never dropped. Air-gapped (a local file '
|
|
466
|
-
+ 'read; sealed-compatible). Files the SCIP index does not cover fall back to heuristic, never '
|
|
467
|
-
+ 'worse. Coverage is printed at index time; re-generate the .scip when code changes.');
|
|
468
|
-
}
|
|
469
|
-
if (config.enrichment.enabled && !config.rerank.enabled) {
|
|
470
|
-
out.push('Enrichment is most effective when paired with --rerank. Running enrichment-only may '
|
|
471
|
-
+ 'regress semantic precision (measured: gin MRR 0.48→0.39 on qwen3 embeddings).');
|
|
472
|
-
}
|
|
473
|
-
if (config.rerank.enabled && config.rerank.provider === 'cross-encoder') {
|
|
474
|
-
out.push('Cross-encoder reranker scores (query, candidate) pairs locally and air-gapped — '
|
|
475
|
-
+ 'no LLM/daemon, deterministic, ~tens of ms per natural-language query. It needs the '
|
|
476
|
-
+ "optional '@huggingface/transformers' package (first run downloads the model). It sees "
|
|
477
|
-
+ 'only a candidate signature/summary line, not full code, so it sharpens near-ties rather '
|
|
478
|
-
+ 'than reasoning about behaviour. Measure on your repo (`npm run test:eval -- --rerank '
|
|
479
|
-
+ '--rerank-provider cross-encoder`) before adopting.');
|
|
480
|
-
} else if (config.rerank.enabled && config.rerank.provider === 'late-interaction') {
|
|
481
|
-
out.push('Late-interaction (ColBERT-style) reranker scores the query MULTI-vectors against '
|
|
482
|
-
+ 'each candidate\'s stored document sub-vectors with MaxSim — air-gapped (the same local '
|
|
483
|
-
+ 'embedder), deterministic, parity-safe, with NO storage blow-up (it reuses the vectors '
|
|
484
|
-
+ '--embeddings already stores). It REQUIRES --embeddings (no doc vectors ⇒ no-op) and '
|
|
485
|
-
+ 'approximates true token-level ColBERT with a sentence embedder over query sub-units. '
|
|
486
|
-
+ 'Measure on your repo (`npm run test:eval -- --embeddings --rerank '
|
|
487
|
-
+ '--rerank-provider late-interaction`) before adopting.');
|
|
488
|
-
} else if (config.rerank.enabled) {
|
|
489
|
-
out.push('Reranker improves rank-1 for Go/Python repositories (gin: 0.20→0.40) but has shown '
|
|
490
|
-
+ 'regressions on JavaScript repositories (express: 0.43→0.29). Behaviour depends on '
|
|
491
|
-
+ 'repository language. See README for per-language benchmark data.');
|
|
492
|
-
}
|
|
493
|
-
return out;
|
|
494
|
-
}
|