graph-indexer 1.2.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +412 -0
- package/README.md +216 -596
- package/SECURITY.md +68 -0
- package/TRADEMARK.md +31 -0
- package/cli-ui.mjs +78 -0
- package/config.mjs +179 -28
- package/daemon-ctl.mjs +175 -0
- package/daemon-lock.mjs +69 -0
- package/embedders/python/mlx_embed_server.py +61 -0
- package/embedders/python/requirements-mlx.txt +5 -0
- package/embedders/setup-mlx.mjs +110 -0
- package/embeddings.mjs +489 -0
- package/engine/binary.mjs +399 -0
- package/{core-engine.mjs → engine/memory.mjs} +107 -440
- package/{sqlite-store.mjs → engine/sqlite.mjs} +137 -42
- package/enrichment.mjs +111 -32
- package/git-signals.mjs +248 -0
- package/indexer.mjs +151 -42
- package/init.mjs +1603 -171
- package/layout.mjs +123 -0
- package/mcp/format.mjs +61 -0
- package/mcp/tools.mjs +1157 -0
- package/mcp/topology.mjs +551 -0
- package/mcp-server.mjs +55 -24
- package/package.json +44 -13
- package/parse/extractor.mjs +617 -0
- package/parse/imports.mjs +365 -0
- package/parse/languages.mjs +89 -0
- package/parse/metadata.mjs +655 -0
- package/parse/routes.mjs +327 -0
- package/prompts/CORE.md +86 -0
- package/prompts/DOMAIN_TEMPLATE.md +46 -0
- package/prompts/INTEGRATION.md +123 -0
- package/prompts/frameworks/ANDROID.md +17 -0
- package/prompts/frameworks/ASPNET_CORE.md +19 -0
- package/prompts/frameworks/FASTAPI_DJANGO.md +19 -0
- package/prompts/frameworks/LARAVEL_SYMFONY.md +22 -0
- package/prompts/frameworks/NODE_EXPRESS_NESTJS.md +18 -0
- package/prompts/frameworks/RAILS.md +20 -0
- package/prompts/frameworks/REACT.md +20 -0
- package/prompts/frameworks/SPRING_BOOT.md +19 -0
- package/prompts/languages/BASH.md +22 -0
- package/prompts/languages/C.md +21 -0
- package/prompts/languages/CSHARP.md +24 -0
- package/prompts/languages/CSS_SCSS.md +22 -0
- package/prompts/languages/GO.md +23 -0
- package/prompts/languages/JAVA.md +23 -0
- package/prompts/languages/JAVASCRIPT_TYPESCRIPT.md +24 -0
- package/prompts/languages/KOTLIN.md +24 -0
- package/prompts/languages/PHP.md +23 -0
- package/prompts/languages/PYTHON.md +23 -0
- package/prompts/languages/RUBY.md +23 -0
- package/prompts/languages/RUST.md +21 -0
- package/prompts/languages/SWIFT.md +21 -0
- package/search-core.mjs +254 -43
- package/storage.mjs +28 -3
- package/watch-daemon.mjs +69 -23
- package/PROMPT.md +0 -182
- package/mcp-tools.mjs +0 -545
- package/parser-utils.mjs +0 -1170
package/SECURITY.md
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Reporting a vulnerability
|
|
4
|
+
|
|
5
|
+
We take the security and privacy of this project seriously. If you discover a
|
|
6
|
+
security vulnerability, please do **not** open a public issue. Instead, report it
|
|
7
|
+
privately by emailing the maintainer at **nicolopezdelerma@gmail.com**.
|
|
8
|
+
|
|
9
|
+
Please include:
|
|
10
|
+
|
|
11
|
+
- A description of the vulnerability and its impact.
|
|
12
|
+
- Steps to reproduce (a minimal proof of concept if possible).
|
|
13
|
+
- The affected version (`graph-indexer --version` or the `version` field in `package.json`).
|
|
14
|
+
|
|
15
|
+
You can expect an initial acknowledgement within **5 business days**. Coordinated
|
|
16
|
+
disclosure is appreciated — we will work with you on a fix and credit before any
|
|
17
|
+
public discussion.
|
|
18
|
+
|
|
19
|
+
## Supported versions
|
|
20
|
+
|
|
21
|
+
Security fixes are applied to the latest published `2.x` release. Older versions
|
|
22
|
+
are not maintained; please upgrade to the latest version.
|
|
23
|
+
|
|
24
|
+
## Security posture
|
|
25
|
+
|
|
26
|
+
graph-indexer is designed to run locally as a developer tool. Its threat model is
|
|
27
|
+
shaped by a few deliberate choices:
|
|
28
|
+
|
|
29
|
+
- **Air-gapped by default.** In normal operation the only outbound network call is to
|
|
30
|
+
a *local* Ollama endpoint (`OLLAMA_HOST`, default `http://localhost:11434`) for
|
|
31
|
+
embeddings. With `INDEXER_EMBEDDINGS=off`, even that call is skipped and the tool is
|
|
32
|
+
fully offline. There are no analytics, telemetry, crash reporters, or third-party API
|
|
33
|
+
calls. **One exception:** the optional in-process embedding provider
|
|
34
|
+
(`embedProvider: "local"`, or `"auto"` when no Ollama is found) downloads its model
|
|
35
|
+
weights from the Hugging Face CDN **once** on first index, then runs fully offline
|
|
36
|
+
from the local cache. Your source code is never sent anywhere — only model weights
|
|
37
|
+
are fetched. For strictly air-gapped installs, set `embedProvider` to `"ollama"` or
|
|
38
|
+
`"off"`, or pre-populate the model cache on a connected machine.
|
|
39
|
+
- **No code execution.** Source files are parsed into ASTs with Tree-sitter. The
|
|
40
|
+
indexer never imports, evaluates, or runs the code it indexes. The one subprocess
|
|
41
|
+
it spawns is a local, read-only `git log` (see "git signals" below) — never the
|
|
42
|
+
indexed code.
|
|
43
|
+
- **Local git signals.** At index time the indexer optionally reads the repository's
|
|
44
|
+
*local* commit log (`git log` on `MCP_PROJECT_ROOT`) to derive churn / recency /
|
|
45
|
+
co-change ranking hints. No remote is contacted and no source content leaves the
|
|
46
|
+
machine — only commit metadata (paths, timestamps) is read locally. Disable it
|
|
47
|
+
entirely with `INDEXER_GIT_SIGNALS=off`, `--no-git-signals`, or
|
|
48
|
+
`"gitSignals": false`.
|
|
49
|
+
- **Path-traversal guard.** The `get_file_skeleton` MCP tool resolves and normalizes
|
|
50
|
+
the requested path and rejects anything outside `MCP_PROJECT_ROOT`. Beyond the
|
|
51
|
+
textual check it also resolves symlinks with `realpath` on both the root and the
|
|
52
|
+
target and re-verifies containment, so a symlink *inside* the project that points
|
|
53
|
+
outside it cannot be used to read arbitrary files on the host.
|
|
54
|
+
- **Local artifacts only.** The index (`code-index.json`) and embeddings
|
|
55
|
+
(`code-index.embeddings.bin`) are written inside the project and are git-ignored by
|
|
56
|
+
`init`. They contain snippets of your source code — treat them with the same
|
|
57
|
+
sensitivity as the repository itself and do not commit or share them.
|
|
58
|
+
- **MCP transport.** The server communicates over stdio with the local MCP client
|
|
59
|
+
(your IDE/agent). It does not open a network socket.
|
|
60
|
+
|
|
61
|
+
## Operational guidance
|
|
62
|
+
|
|
63
|
+
- Point `MCP_PROJECT_ROOT` only at repositories you trust and intend to index.
|
|
64
|
+
- If you set `OLLAMA_HOST` to a remote host, that host will receive code snippets as
|
|
65
|
+
embedding input — only do so on a network and host you control.
|
|
66
|
+
- The background watch daemon respects `.gitignore` and skips `node_modules`, build
|
|
67
|
+
output, and dot-directories, so secrets in ignored paths are not indexed. Keep
|
|
68
|
+
sensitive files git-ignored.
|
package/TRADEMARK.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Trademark Policy
|
|
2
|
+
|
|
3
|
+
This document outlines the trademark policy for the **graph-indexer** project and the **MaquinaTech** brand.
|
|
4
|
+
|
|
5
|
+
While the software source code is freely available under the [MIT License](LICENSE), the trademarks, service marks, and logos associated with the project are the intellectual property of MaquinaTech. The open-source license does not grant you the right to use these trademarks in a way that suggests endorsement, affiliation, or origin.
|
|
6
|
+
|
|
7
|
+
## Our Trademarks
|
|
8
|
+
|
|
9
|
+
The following are trademarks of MaquinaTech:
|
|
10
|
+
* **graph-indexer™**
|
|
11
|
+
* **Graph Indexer™**
|
|
12
|
+
* **MaquinaTech™**
|
|
13
|
+
|
|
14
|
+
## Permitted Uses
|
|
15
|
+
|
|
16
|
+
You may use our trademarks without prior written permission in the following cases:
|
|
17
|
+
* **Descriptive Use:** To truthfully describe the origin of the software. For example, stating "This product uses graph-indexer."
|
|
18
|
+
* **Compatibility:** To indicate compatibility with our project. For example, "Custom tool for graph-indexer," provided it does not confuse users into thinking it is an official tool created by us.
|
|
19
|
+
* **Community Discussion:** In blog posts, tutorials, videos, and discussions about the project, provided the context is accurate and not misleading.
|
|
20
|
+
|
|
21
|
+
## Restricted Uses
|
|
22
|
+
|
|
23
|
+
You **may not** use our trademarks without explicit, prior written permission in the following ways:
|
|
24
|
+
* **Product Naming:** You cannot name your product, commercial service, or company "graph-indexer", "Graph Indexer", or anything confusingly similar (e.g., "Graph Indexer Enterprise", "Graph-Indexer Pro", or "graph-indexer-cloud").
|
|
25
|
+
* **Domain Names:** You cannot use our trademarks in a domain name (e.g., `graph-indexer-hosting.com` or `maquinatech-solutions.net`).
|
|
26
|
+
* **False Endorsement:** You cannot use our trademarks in any way that implies an official affiliation, partnership, sponsorship, or endorsement by MaquinaTech.
|
|
27
|
+
* **Modified Versions (Forks):** If you fork the repository and make modifications to the source code to distribute it as your own product, you must rename the project. You cannot distribute your modified version under the name "graph-indexer".
|
|
28
|
+
|
|
29
|
+
## Contact and Permissions
|
|
30
|
+
|
|
31
|
+
If you wish to use our trademarks in a way that is not explicitly permitted by this policy, or if you are unsure whether your intended commercial use is permitted, please contact us for written authorization.
|
package/cli-ui.mjs
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file cli-ui.mjs
|
|
3
|
+
* @description Tiny, dependency-free console styling shared by the `graph-indexer`
|
|
4
|
+
* CLIs (init + daemon control). ANSI colour with graceful degradation
|
|
5
|
+
* when stdout is not a TTY or NO_COLOR is set, plus a few drawing
|
|
6
|
+
* helpers (boxed banner, rules, status glyphs) for a consistent look.
|
|
7
|
+
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
8
|
+
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
9
|
+
* @license MIT
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export const useColor = process.stdout.isTTY && !process.env.NO_COLOR;
|
|
13
|
+
|
|
14
|
+
const paint = (code) => (s) => (useColor ? `\x1b[${code}m${s}\x1b[0m` : `${s}`);
|
|
15
|
+
|
|
16
|
+
export const c = {
|
|
17
|
+
bold: paint('1'),
|
|
18
|
+
dim: paint('2'),
|
|
19
|
+
red: paint('31'),
|
|
20
|
+
green: paint('32'),
|
|
21
|
+
yellow: paint('33'),
|
|
22
|
+
blue: paint('34'),
|
|
23
|
+
magenta: paint('35'),
|
|
24
|
+
cyan: paint('36'),
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export const glyph = {
|
|
28
|
+
ok: c.green('✓'),
|
|
29
|
+
upd: c.cyan('↻'),
|
|
30
|
+
keep: c.dim('•'),
|
|
31
|
+
move: c.yellow('↪'),
|
|
32
|
+
skip: c.dim('–'),
|
|
33
|
+
warn: c.yellow('⚠'),
|
|
34
|
+
err: c.red('✗'),
|
|
35
|
+
run: c.green('●'),
|
|
36
|
+
stop: c.dim('○'),
|
|
37
|
+
arrow: c.cyan('→'),
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export const log = (msg = '') => process.stdout.write(msg + '\n');
|
|
41
|
+
|
|
42
|
+
/** A horizontal rule, capped to the terminal width so it never wraps. */
|
|
43
|
+
export const rule = (width = Math.min(60, (process.stdout.columns || 80))) => c.dim('─'.repeat(Math.max(1, width)));
|
|
44
|
+
|
|
45
|
+
const visibleLen = (s) => s.replace(/\x1b\[[0-9;]*m/g, '').length;
|
|
46
|
+
|
|
47
|
+
/** ANSI-aware truncate to `limit` visible columns; closes any open colour + adds an ellipsis. */
|
|
48
|
+
function clipVisible(s, limit) {
|
|
49
|
+
if (limit < 1) return '';
|
|
50
|
+
if (visibleLen(s) <= limit) return s;
|
|
51
|
+
let out = '', w = 0, i = 0, ansi = false;
|
|
52
|
+
while (i < s.length && w < limit - 1) {
|
|
53
|
+
if (s[i] === '\x1b') {
|
|
54
|
+
const m = s.slice(i).match(/^\x1b\[[0-9;]*m/);
|
|
55
|
+
if (m) { out += m[0]; i += m[0].length; ansi = true; continue; }
|
|
56
|
+
}
|
|
57
|
+
out += s[i++]; w++;
|
|
58
|
+
}
|
|
59
|
+
return out + (ansi ? '\x1b[0m' : '') + '…';
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Draws a bordered box. ANSI escapes are stripped before measuring width so colour
|
|
64
|
+
* never breaks alignment, and the box (plus each line) is capped to the terminal
|
|
65
|
+
* width so the border stays intact on narrow terminals instead of wrapping.
|
|
66
|
+
*/
|
|
67
|
+
export function box(lines, { pad = 1, color = c.cyan } = {}) {
|
|
68
|
+
const maxInner = Math.max(1, (process.stdout.columns || 80) - 2); // 2 border columns
|
|
69
|
+
const shown = lines.map((line) => clipVisible(line, maxInner - pad * 2));
|
|
70
|
+
const inner = Math.min(maxInner, Math.max(...shown.map(visibleLen)) + pad * 2);
|
|
71
|
+
const top = color('╭' + '─'.repeat(inner) + '╮');
|
|
72
|
+
const bot = color('╰' + '─'.repeat(inner) + '╯');
|
|
73
|
+
const body = shown.map((line) => {
|
|
74
|
+
const gap = ' '.repeat(Math.max(0, inner - visibleLen(line) - pad));
|
|
75
|
+
return color('│') + ' '.repeat(pad) + line + gap + color('│');
|
|
76
|
+
});
|
|
77
|
+
return [top, ...body, bot].join('\n');
|
|
78
|
+
}
|
package/config.mjs
CHANGED
|
@@ -10,32 +10,75 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import fs from 'fs';
|
|
12
12
|
import path from 'path';
|
|
13
|
+
import { artifactPaths, CONFIG_FILE_NAME, DATA_DIR_NAME } from './layout.mjs';
|
|
13
14
|
|
|
14
15
|
export const DEFAULTS = Object.freeze({
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
// 'auto' keeps the index in memory below AUTO_SQLITE_CHUNK_THRESHOLD chunks and
|
|
17
|
+
// switches to disk-backed SQLite above it. Force either with --use-sqlite /
|
|
18
|
+
// INDEXER_STORAGE. (see storage.mjs: resolveBackend / AUTO_SQLITE_CHUNK_THRESHOLD)
|
|
19
|
+
storage: 'auto', // 'auto' (default) | 'memory' | 'sqlite'
|
|
20
|
+
embeddings: false, // OFF by default — lexical + stemming need zero dependencies.
|
|
21
|
+
// Opt in with --embeddings or INDEXER_EMBEDDINGS=on.
|
|
22
|
+
embedProvider: 'auto',
|
|
23
|
+
// 'auto' → Ollama (if running w/ the model) → in-process local → lexical-only
|
|
24
|
+
// 'ollama' → force Ollama daemon (requires: ollama serve + pulled model)
|
|
25
|
+
// 'local' → in-process Xenova (requires: npm i @huggingface/transformers)
|
|
26
|
+
// 'mlx' → Apple Metal GPU (requires: npm run embed:setup:mlx; macOS only)
|
|
27
|
+
// 'off' → lexical-only, no vectors
|
|
28
|
+
embedModel: 'nomic-embed-text', // default Ollama embed model when embeddings are enabled.
|
|
29
|
+
// qwen3-embedding:4b is a documented opt-in upgrade
|
|
30
|
+
// (--embed-model / EMBED_MODEL), never a hardcoded default.
|
|
31
|
+
localEmbedModel: 'Xenova/all-MiniLM-L6-v2', // in-process model (optional @huggingface/transformers)
|
|
32
|
+
mlxEmbedModel: 'mlx-community/all-MiniLM-L6-v2-4bit', // MLX embedder model (provider 'mlx').
|
|
33
|
+
// Override with --mlx-embed-model / INDEXER_MLX_EMBED_MODEL;
|
|
34
|
+
// must be an mlx_embeddings-compatible sentence model.
|
|
17
35
|
ollamaHost: 'http://localhost:11434',
|
|
36
|
+
llmProvider: 'ollama', // LLM backend for enrichment, reranking, HyDE: 'ollama' | 'mlx'
|
|
37
|
+
// 'mlx' requires mlx_lm.server running (mlx_lm.server --model <name>)
|
|
38
|
+
mlxLmHost: 'http://localhost:8080', // mlx_lm.server endpoint (default port 8080)
|
|
39
|
+
gitSignals: true, // collect local git churn/recency/co-change at index time (air-gapped)
|
|
40
|
+
gitRankBoost: 0, // 0..1 opt-in recency/churn weight in search_code (0 = ranking unchanged)
|
|
18
41
|
enrichment: Object.freeze({
|
|
19
42
|
enabled: false,
|
|
20
|
-
model: 'qwen2.5-coder:1.5b', //
|
|
43
|
+
model: 'qwen2.5-coder:1.5b', // smallest coder model; quality sufficient for summaries.
|
|
44
|
+
// Opt in with --enrichment or ENRICH_MODEL=<model>.
|
|
21
45
|
coreRatio: 1.0, // 1.0 = all production files (tests/examples always excluded);
|
|
22
46
|
// <1 bounds enrichment to the most-central share by PageRank
|
|
23
47
|
maxChunks: 500, // cap on NEW LLM calls per index run (cache accumulates across runs)
|
|
24
|
-
concurrency:
|
|
48
|
+
concurrency: 4, // parallel Ollama requests during enrichment; keep low —
|
|
49
|
+
// a single local model serves requests fastest one-at-a-time
|
|
50
|
+
// unless OLLAMA_NUM_PARALLEL + hardware allow more
|
|
51
|
+
|
|
25
52
|
}),
|
|
26
53
|
rerank: Object.freeze({
|
|
27
|
-
enabled: false, // opt-in: one LLM call
|
|
54
|
+
enabled: false, // opt-in: one LLM call per natural-language query
|
|
28
55
|
model: 'qwen2.5-coder:7b', // judge quality matters: 7B measured +50% semantic rank-1, 1.5B ~nil
|
|
29
|
-
topM:
|
|
56
|
+
topM: 12, // candidates shown to the judge to reorder
|
|
57
|
+
poolSize: 15, // over-fetch depth when reranking, so a correct-but-deep
|
|
58
|
+
// hit can be RESCUED into top_k (then truncated to top_k)
|
|
59
|
+
}),
|
|
60
|
+
hyde: Object.freeze({
|
|
61
|
+
enabled: false, // opt-in: query-side HyDE — one LLM call per NL query, blended into the query vector
|
|
62
|
+
model: 'qwen2.5-coder:1.5b', // local coder model that writes the hypothetical snippet
|
|
30
63
|
}),
|
|
31
64
|
});
|
|
32
65
|
|
|
33
|
-
/**
|
|
66
|
+
/**
|
|
67
|
+
* Reads the project config, tolerating absence/corruption. Looks first inside the
|
|
68
|
+
* tidy data dir (`.graph-indexer/config.json`), then falls back to the legacy
|
|
69
|
+
* root file (`.graph-indexer.json`) so pre-v1.4 projects keep working until they
|
|
70
|
+
* are migrated.
|
|
71
|
+
*/
|
|
34
72
|
export function loadConfigFile(root) {
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
73
|
+
const candidates = [
|
|
74
|
+
path.join(root, DATA_DIR_NAME, CONFIG_FILE_NAME),
|
|
75
|
+
path.join(root, '.graph-indexer.json'), // legacy root config (back-compat)
|
|
76
|
+
];
|
|
77
|
+
for (const configPath of candidates) {
|
|
78
|
+
try {
|
|
79
|
+
if (fs.existsSync(configPath)) return JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
|
80
|
+
} catch { /* malformed config — try the next candidate, else defaults */ }
|
|
81
|
+
}
|
|
39
82
|
return {};
|
|
40
83
|
}
|
|
41
84
|
|
|
@@ -63,36 +106,85 @@ export function resolveConfig({ argv = process.argv.slice(2), env = process.env,
|
|
|
63
106
|
const file = loadConfigFile(projectRoot);
|
|
64
107
|
const fileEnrich = file.enrichment || {};
|
|
65
108
|
|
|
66
|
-
// Storage: --use-sqlite
|
|
67
|
-
//
|
|
68
|
-
|
|
69
|
-
|
|
109
|
+
// Storage: --use-sqlite forces SQLite; otherwise INDEXER_STORAGE / "storage" key
|
|
110
|
+
// / default 'auto' (resolved to memory-or-sqlite by chunk count at index time and
|
|
111
|
+
// by artifact presence at query time — see storage.mjs:resolveBackend).
|
|
112
|
+
const storageRaw = argv.includes('--use-sqlite') ? 'sqlite'
|
|
113
|
+
: (env.INDEXER_STORAGE || file.storage || DEFAULTS.storage);
|
|
114
|
+
const storage = ['memory', 'sqlite', 'auto'].includes(storageRaw) ? storageRaw : DEFAULTS.storage;
|
|
70
115
|
|
|
71
|
-
|
|
72
|
-
|
|
116
|
+
// Enrichment is OFF unless explicitly requested: --enrichment flag, ENRICH_MODEL
|
|
117
|
+
// env (naming a model implies enabling it), or the project config.
|
|
118
|
+
const enrichModelEnv = env.ENRICH_MODEL || undefined;
|
|
119
|
+
const enrichmentEnabled = argv.includes('--enrichment')
|
|
120
|
+
|| Boolean(enrichModelEnv)
|
|
73
121
|
|| Boolean(fileEnrich.enabled);
|
|
74
122
|
|
|
75
123
|
const ollamaHost = env.OLLAMA_HOST || file.ollamaHost || DEFAULTS.ollamaHost;
|
|
76
|
-
|
|
124
|
+
|
|
125
|
+
// LLM provider for enrichment / reranking / HyDE: env > flag > config > default.
|
|
126
|
+
// 'mlx' routes generation through a local mlx_lm.server (OpenAI-compatible).
|
|
127
|
+
const llmProviderRaw = env.INDEXER_LLM_PROVIDER || flagValue(argv, '--llm-provider') || file.llmProvider || DEFAULTS.llmProvider;
|
|
128
|
+
const llmProvider = ['ollama', 'mlx'].includes(llmProviderRaw) ? llmProviderRaw : DEFAULTS.llmProvider;
|
|
129
|
+
const mlxLmHost = env.INDEXER_MLX_LM_HOST || flagValue(argv, '--mlx-lm-host') || file.mlxLmHost || DEFAULTS.mlxLmHost;
|
|
130
|
+
|
|
131
|
+
// Embeddings are OFF by default (lexical + stemming need zero dependencies).
|
|
132
|
+
// Enable with --embeddings or INDEXER_EMBEDDINGS=on; INDEXER_EMBEDDINGS=off always wins.
|
|
133
|
+
const embeddingsEnabled = env.INDEXER_EMBEDDINGS === 'off'
|
|
134
|
+
? false
|
|
135
|
+
: (argv.includes('--embeddings') || env.INDEXER_EMBEDDINGS === 'on' || file.embeddings === true);
|
|
136
|
+
|
|
137
|
+
// Embedding provider: env > --embed-provider flag > config > default 'auto'.
|
|
138
|
+
// 'auto' prefers a running Ollama and falls back to the in-process local model
|
|
139
|
+
// (optional @huggingface/transformers) so semantic search works with no daemon.
|
|
140
|
+
const embedProvider = env.INDEXER_EMBED_PROVIDER
|
|
141
|
+
|| flagValue(argv, '--embed-provider')
|
|
142
|
+
|| file.embedProvider || DEFAULTS.embedProvider;
|
|
143
|
+
|
|
144
|
+
// Git signals: collection is air-gapped (local `git log` only). Disable with
|
|
145
|
+
// INDEXER_GIT_SIGNALS=off, --no-git-signals, or "gitSignals": false.
|
|
146
|
+
const gitSignals = env.INDEXER_GIT_SIGNALS === 'off' || argv.includes('--no-git-signals')
|
|
147
|
+
? false
|
|
148
|
+
: (file.gitSignals === false ? false : DEFAULTS.gitSignals);
|
|
149
|
+
const gitRankBoostRaw = env.INDEXER_GIT_RANK_BOOST ?? flagValue(argv, '--git-rank-boost')
|
|
150
|
+
?? (Number.isFinite(file.gitRankBoost) ? file.gitRankBoost : DEFAULTS.gitRankBoost);
|
|
151
|
+
const gitRankBoost = Math.min(1, Math.max(0, Number(gitRankBoostRaw) || 0));
|
|
152
|
+
|
|
153
|
+
const paths = artifactPaths(projectRoot);
|
|
77
154
|
|
|
78
155
|
return Object.freeze({
|
|
79
156
|
projectRoot,
|
|
80
157
|
storage,
|
|
81
|
-
|
|
82
|
-
indexPath:
|
|
83
|
-
embeddingPath:
|
|
84
|
-
sqlitePath:
|
|
85
|
-
enrichmentCachePath:
|
|
158
|
+
dataDir: paths.dataDir,
|
|
159
|
+
indexPath: paths.indexPath,
|
|
160
|
+
embeddingPath: paths.embeddingPath,
|
|
161
|
+
sqlitePath: paths.sqlitePath,
|
|
162
|
+
enrichmentCachePath: paths.enrichmentCachePath,
|
|
163
|
+
gitSignalsPath: paths.gitSignalsPath,
|
|
164
|
+
pidFile: paths.pidFile,
|
|
165
|
+
logFile: paths.logFile,
|
|
86
166
|
|
|
87
167
|
languages: Array.isArray(file.languages) ? file.languages : null, // null = all
|
|
88
168
|
|
|
89
169
|
ollamaHost,
|
|
170
|
+
llmProvider,
|
|
171
|
+
mlxLmHost,
|
|
90
172
|
embeddingsEnabled,
|
|
91
|
-
|
|
173
|
+
embedProvider,
|
|
174
|
+
// Embed model: --embed-model flag > EMBED_MODEL env > config > nomic default.
|
|
175
|
+
// This is the model `auto`/`ollama` providers pull (e.g. qwen3-embedding:4b).
|
|
176
|
+
embedModel: flagValue(argv, '--embed-model') || env.EMBED_MODEL || file.embedModel || DEFAULTS.embedModel,
|
|
177
|
+
localEmbedModel: file.localEmbedModel || DEFAULTS.localEmbedModel,
|
|
178
|
+
// MLX embedder model: --mlx-embed-model flag > INDEXER_MLX_EMBED_MODEL env >
|
|
179
|
+
// config > pinned MiniLM default. Passed to the Python server + stamped in meta.
|
|
180
|
+
mlxEmbedModel: flagValue(argv, '--mlx-embed-model') || env.INDEXER_MLX_EMBED_MODEL
|
|
181
|
+
|| file.mlxEmbedModel || DEFAULTS.mlxEmbedModel,
|
|
182
|
+
gitSignals,
|
|
183
|
+
gitRankBoost,
|
|
92
184
|
|
|
93
185
|
enrichment: Object.freeze({
|
|
94
186
|
enabled: enrichmentEnabled,
|
|
95
|
-
model: flagValue(argv, '--enrich-model') || fileEnrich.model || DEFAULTS.enrichment.model,
|
|
187
|
+
model: flagValue(argv, '--enrich-model') || enrichModelEnv || fileEnrich.model || DEFAULTS.enrichment.model,
|
|
96
188
|
coreRatio: Number(fileEnrich.coreRatio) > 0 ? Number(fileEnrich.coreRatio) : DEFAULTS.enrichment.coreRatio,
|
|
97
189
|
maxChunks: Number(flagValue(argv, '--enrich-max')) > 0
|
|
98
190
|
? Number(flagValue(argv, '--enrich-max'))
|
|
@@ -103,16 +195,75 @@ export function resolveConfig({ argv = process.argv.slice(2), env = process.env,
|
|
|
103
195
|
}),
|
|
104
196
|
|
|
105
197
|
rerank: Object.freeze({
|
|
106
|
-
|
|
107
|
-
|
|
198
|
+
// OFF unless requested: --rerank flag, RERANK_MODEL env (naming a model
|
|
199
|
+
// implies enabling it), or the project config.
|
|
200
|
+
enabled: argv.includes('--rerank') || Boolean(env.RERANK_MODEL) || Boolean((file.rerank || {}).enabled),
|
|
201
|
+
model: env.RERANK_MODEL || (file.rerank || {}).model || DEFAULTS.rerank.model,
|
|
108
202
|
topM: Number.isInteger((file.rerank || {}).topM) ? (file.rerank || {}).topM : DEFAULTS.rerank.topM,
|
|
203
|
+
poolSize: Number.isInteger((file.rerank || {}).poolSize) ? (file.rerank || {}).poolSize : DEFAULTS.rerank.poolSize,
|
|
204
|
+
}),
|
|
205
|
+
|
|
206
|
+
hyde: Object.freeze({
|
|
207
|
+
enabled: Boolean((file.hyde || {}).enabled),
|
|
208
|
+
model: (file.hyde || {}).model || DEFAULTS.hyde.model,
|
|
109
209
|
}),
|
|
110
210
|
});
|
|
111
211
|
}
|
|
112
212
|
|
|
113
|
-
// Memoised singleton for import-time consumers (e.g.
|
|
213
|
+
// Memoised singleton for import-time consumers (e.g. parse/languages.mjs language loading).
|
|
114
214
|
let _cached = null;
|
|
115
215
|
export function getConfig() {
|
|
116
216
|
if (!_cached) _cached = resolveConfig();
|
|
117
217
|
return _cached;
|
|
118
218
|
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Human-readable lines describing the effective configuration, printed at startup
|
|
222
|
+
* by the indexer / MCP server / daemon so users can see exactly what is running
|
|
223
|
+
* (storage backend, model names, which optional features are on).
|
|
224
|
+
*
|
|
225
|
+
* @param {object} config Resolved config from resolveConfig().
|
|
226
|
+
* @param {object} [opts]
|
|
227
|
+
* @param {string} [opts.backend] Concrete backend ('memory'|'sqlite') once 'auto'
|
|
228
|
+
* has been resolved; defaults to config.storage.
|
|
229
|
+
* @returns {string[]}
|
|
230
|
+
*/
|
|
231
|
+
export function describeConfig(config, { backend = config.storage } = {}) {
|
|
232
|
+
// Show the model that the chosen provider actually loads (mlx/local use their own).
|
|
233
|
+
const provModel = config.embedProvider === 'mlx' ? config.mlxEmbedModel
|
|
234
|
+
: config.embedProvider === 'local' ? config.localEmbedModel
|
|
235
|
+
: config.embedModel;
|
|
236
|
+
const emb = config.embeddingsEnabled
|
|
237
|
+
? `on · provider=${config.embedProvider}, model=${provModel}`
|
|
238
|
+
: 'off (lexical + stemming only)';
|
|
239
|
+
return [
|
|
240
|
+
`storage : ${backend}`,
|
|
241
|
+
`embeddings : ${emb}`,
|
|
242
|
+
`enrichment : ${config.enrichment.enabled ? `on · model=${config.enrichment.model} · provider=${config.llmProvider}` : 'off'}`,
|
|
243
|
+
`reranker : ${config.rerank.enabled ? `on · model=${config.rerank.model} · provider=${config.llmProvider}` : 'off'}`,
|
|
244
|
+
`git signals : ${config.gitSignals ? 'on' : 'off'}${config.gitRankBoost ? ` · rank boost=${config.gitRankBoost}` : ''}`,
|
|
245
|
+
];
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Operational warnings for risky / edge configurations. The transition between
|
|
250
|
+
* default and an opt-in feature must never be silent: callers print these at
|
|
251
|
+
* startup so a user who enabled enrichment-only or the reranker sees the measured
|
|
252
|
+
* trade-off before they wonder why precision moved.
|
|
253
|
+
*
|
|
254
|
+
* @param {object} config Resolved config from resolveConfig().
|
|
255
|
+
* @returns {string[]}
|
|
256
|
+
*/
|
|
257
|
+
export function configNotices(config) {
|
|
258
|
+
const out = [];
|
|
259
|
+
if (config.enrichment.enabled && !config.rerank.enabled) {
|
|
260
|
+
out.push('Enrichment is most effective when paired with --rerank. Running enrichment-only may '
|
|
261
|
+
+ 'regress semantic precision (measured: gin MRR 0.48→0.39 on qwen3 embeddings).');
|
|
262
|
+
}
|
|
263
|
+
if (config.rerank.enabled) {
|
|
264
|
+
out.push('Reranker improves rank-1 for Go/Python repositories (gin: 0.20→0.40) but has shown '
|
|
265
|
+
+ 'regressions on JavaScript repositories (express: 0.43→0.29). Behaviour depends on '
|
|
266
|
+
+ 'repository language. See README for per-language benchmark data.');
|
|
267
|
+
}
|
|
268
|
+
return out;
|
|
269
|
+
}
|
package/daemon-ctl.mjs
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @file daemon-ctl.mjs
|
|
4
|
+
* @description `idx-daemon` — control the graph-indexer watch daemon from the
|
|
5
|
+
* command line (and therefore from npm scripts): start, stop,
|
|
6
|
+
* restart, status, logs. Exactly one daemon runs per project; the
|
|
7
|
+
* daemon self-guards via an atomic PID lock (daemon-lock.mjs), so
|
|
8
|
+
* `start` is safe to call repeatedly and from any launcher.
|
|
9
|
+
*
|
|
10
|
+
* idx-daemon start Start it (no-op if already running)
|
|
11
|
+
* idx-daemon stop Gracefully stop it
|
|
12
|
+
* idx-daemon restart Stop then start
|
|
13
|
+
* idx-daemon status Show daemon + index state (default)
|
|
14
|
+
* idx-daemon logs [-f] Print recent daemon logs (-f to follow)
|
|
15
|
+
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
16
|
+
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
17
|
+
* @license MIT
|
|
18
|
+
*/
|
|
19
|
+
import fs from 'fs';
|
|
20
|
+
import path from 'path';
|
|
21
|
+
import { fileURLToPath } from 'url';
|
|
22
|
+
import { spawn } from 'child_process';
|
|
23
|
+
import { resolveConfig } from './config.mjs';
|
|
24
|
+
import { resolveBackend } from './storage.mjs';
|
|
25
|
+
import { ensureDataDir, migrateLegacyLayout } from './layout.mjs';
|
|
26
|
+
import { daemonStatus, readPid, isAlive } from './daemon-lock.mjs';
|
|
27
|
+
import { c, glyph, log } from './cli-ui.mjs';
|
|
28
|
+
|
|
29
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
30
|
+
const DAEMON_PATH = path.join(__dirname, 'watch-daemon.mjs');
|
|
31
|
+
|
|
32
|
+
const config = resolveConfig();
|
|
33
|
+
const ROOT = config.projectRoot;
|
|
34
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
35
|
+
|
|
36
|
+
function relArtifact() {
|
|
37
|
+
const p = resolveBackend(config) === 'sqlite' ? config.sqlitePath : config.indexPath;
|
|
38
|
+
return path.relative(ROOT, p) || p;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function freshness(file) {
|
|
42
|
+
try {
|
|
43
|
+
const sec = Math.floor((Date.now() - fs.statSync(file).mtimeMs) / 1000);
|
|
44
|
+
if (sec < 60) return `${sec}s ago`;
|
|
45
|
+
if (sec < 3600) return `${Math.floor(sec / 60)}m ago`;
|
|
46
|
+
if (sec < 86400) return `${Math.floor(sec / 3600)}h ago`;
|
|
47
|
+
return `${Math.floor(sec / 86400)}d ago`;
|
|
48
|
+
} catch { return 'never indexed'; }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function spawnDaemon() {
|
|
52
|
+
ensureDataDir(ROOT);
|
|
53
|
+
let logFd = null;
|
|
54
|
+
try { logFd = fs.openSync(config.logFile, 'a'); } catch { }
|
|
55
|
+
const child = spawn(process.execPath, [DAEMON_PATH], {
|
|
56
|
+
detached: true,
|
|
57
|
+
stdio: logFd !== null ? ['ignore', logFd, logFd] : 'ignore',
|
|
58
|
+
env: { ...process.env, MCP_PROJECT_ROOT: ROOT },
|
|
59
|
+
});
|
|
60
|
+
child.unref();
|
|
61
|
+
if (logFd !== null) fs.closeSync(logFd);
|
|
62
|
+
return child.pid;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function cmdStatus() {
|
|
66
|
+
const { running, pid } = daemonStatus(config.pidFile);
|
|
67
|
+
const backend = resolveBackend(config);
|
|
68
|
+
const artifact = backend === 'sqlite' ? config.sqlitePath : config.indexPath;
|
|
69
|
+
log('');
|
|
70
|
+
log(` ${c.bold('graph-indexer')} ${c.dim('· daemon status')}`);
|
|
71
|
+
log(` ${running ? glyph.run : glyph.stop} Daemon ${running ? c.green(`running (PID ${pid})`) : c.dim('stopped')}`);
|
|
72
|
+
log(` ${glyph.arrow} Project ${c.dim(ROOT)}`);
|
|
73
|
+
log(` ${glyph.arrow} Backend ${backend === 'sqlite' ? 'SQLite (disk-backed)' : 'in-memory'}${config.storage === 'auto' ? c.dim(' · auto') : ''}`);
|
|
74
|
+
log(` ${glyph.arrow} Index ${c.dim(relArtifact())} ${c.dim('· ' + freshness(artifact))}`);
|
|
75
|
+
log(` ${glyph.arrow} Logs ${c.dim(path.relative(ROOT, config.logFile) || config.logFile)}`);
|
|
76
|
+
log('');
|
|
77
|
+
if (!running) log(c.dim(` Start it with: ${c.cyan('npm run mcp:daemon:start')}\n`));
|
|
78
|
+
return 0;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function cmdStart() {
|
|
82
|
+
ensureDataDir(ROOT);
|
|
83
|
+
migrateLegacyLayout(ROOT);
|
|
84
|
+
const { running, pid } = daemonStatus(config.pidFile);
|
|
85
|
+
if (running) {
|
|
86
|
+
log(`${glyph.keep} Daemon already running ${c.dim(`(PID ${pid})`)} — only one runs per project.`);
|
|
87
|
+
return 0;
|
|
88
|
+
}
|
|
89
|
+
const newPid = spawnDaemon();
|
|
90
|
+
log(`${glyph.ok} Watch daemon started ${c.dim(`(PID ${newPid})`)}`);
|
|
91
|
+
log(c.dim(` Logs: ${path.relative(ROOT, config.logFile) || config.logFile}`));
|
|
92
|
+
return 0;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async function cmdStop() {
|
|
96
|
+
const pid = readPid(config.pidFile);
|
|
97
|
+
if (!isAlive(pid)) {
|
|
98
|
+
log(`${glyph.keep} No daemon running.`);
|
|
99
|
+
try { if (pid) fs.unlinkSync(config.pidFile); } catch { }
|
|
100
|
+
return 0;
|
|
101
|
+
}
|
|
102
|
+
try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ }
|
|
103
|
+
for (let i = 0; i < 50 && isAlive(readPid(config.pidFile) ?? pid); i++) await sleep(100);
|
|
104
|
+
if (isAlive(pid)) {
|
|
105
|
+
log(`${glyph.warn} Daemon ${c.dim(`(PID ${pid})`)} did not stop within 5s.`);
|
|
106
|
+
return 1;
|
|
107
|
+
}
|
|
108
|
+
log(`${glyph.ok} Daemon stopped ${c.dim(`(PID ${pid})`)}.`);
|
|
109
|
+
return 0;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function cmdRestart() {
|
|
113
|
+
await cmdStop();
|
|
114
|
+
await sleep(200);
|
|
115
|
+
return cmdStart();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function tail(file, n) {
|
|
119
|
+
try {
|
|
120
|
+
const size = fs.statSync(file).size;
|
|
121
|
+
const start = Math.max(0, size - 256 * 1024); // last 256 KB is plenty
|
|
122
|
+
const fd = fs.openSync(file, 'r');
|
|
123
|
+
try {
|
|
124
|
+
const buf = Buffer.alloc(size - start);
|
|
125
|
+
fs.readSync(fd, buf, 0, buf.length, start);
|
|
126
|
+
const lines = buf.toString('utf-8').split('\n');
|
|
127
|
+
return { lines: lines.slice(-n - 1), size };
|
|
128
|
+
} finally { fs.closeSync(fd); }
|
|
129
|
+
} catch { return { lines: [], size: 0 }; }
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async function cmdLogs(args) {
|
|
133
|
+
const follow = args.includes('-f') || args.includes('--follow');
|
|
134
|
+
const nArg = args.find((a) => /^\d+$/.test(a));
|
|
135
|
+
const n = nArg ? Number(nArg) : 50;
|
|
136
|
+
if (!fs.existsSync(config.logFile)) {
|
|
137
|
+
log(c.dim(`No daemon log yet (${path.relative(ROOT, config.logFile) || config.logFile}).`));
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
140
|
+
const { lines, size } = tail(config.logFile, n);
|
|
141
|
+
process.stdout.write(lines.join('\n'));
|
|
142
|
+
if (!follow) { log(''); return 0; }
|
|
143
|
+
|
|
144
|
+
let pos = size;
|
|
145
|
+
log(c.dim(`\n— following ${path.relative(ROOT, config.logFile)} (Ctrl-C to stop) —`));
|
|
146
|
+
fs.watchFile(config.logFile, { interval: 500 }, (cur) => {
|
|
147
|
+
if (cur.size < pos) { pos = 0; return; } // truncated/rotated
|
|
148
|
+
if (cur.size === pos) return;
|
|
149
|
+
const fd = fs.openSync(config.logFile, 'r');
|
|
150
|
+
try {
|
|
151
|
+
const buf = Buffer.alloc(cur.size - pos);
|
|
152
|
+
fs.readSync(fd, buf, 0, buf.length, pos);
|
|
153
|
+
process.stdout.write(buf.toString('utf-8'));
|
|
154
|
+
pos = cur.size;
|
|
155
|
+
} finally { fs.closeSync(fd); }
|
|
156
|
+
});
|
|
157
|
+
return new Promise(() => { /* runs until Ctrl-C */ });
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const [cmd = 'status', ...rest] = process.argv.slice(2);
|
|
161
|
+
const run = {
|
|
162
|
+
start: cmdStart,
|
|
163
|
+
stop: cmdStop,
|
|
164
|
+
restart: cmdRestart,
|
|
165
|
+
status: cmdStatus,
|
|
166
|
+
logs: () => cmdLogs(rest),
|
|
167
|
+
}[cmd];
|
|
168
|
+
|
|
169
|
+
if (!run) {
|
|
170
|
+
log(`${glyph.err} Unknown command: ${c.bold(cmd)}`);
|
|
171
|
+
log(c.dim(' Usage: idx-daemon [start|stop|restart|status|logs]'));
|
|
172
|
+
process.exit(2);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
Promise.resolve(run()).then((code) => process.exit(code ?? 0));
|