edhindex 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.edhindexignore +66 -0
- package/BUILD_BRIEF.md +228 -0
- package/package.json +38 -0
- package/src/cli/commands/config_cmd.ts +58 -0
- package/src/cli/commands/doctor_cmd.ts +31 -0
- package/src/cli/commands/index_cmd.ts +121 -0
- package/src/cli/commands/init.ts +25 -0
- package/src/cli/commands/models.ts +19 -0
- package/src/cli/commands/search.ts +44 -0
- package/src/cli/commands/start.ts +253 -0
- package/src/cli/commands/status.ts +54 -0
- package/src/cli/index.ts +99 -0
- package/src/config/index.ts +75 -0
- package/src/doctor/index.ts +131 -0
- package/src/embeddings/provider.ts +9 -0
- package/src/embeddings/transformers.ts +130 -0
- package/src/indexer/chunker.ts +198 -0
- package/src/indexer/file-utils.ts +139 -0
- package/src/indexer/ignore.ts +100 -0
- package/src/indexer/incremental.ts +80 -0
- package/src/indexer/indexer.ts +181 -0
- package/src/indexer/parser.ts +267 -0
- package/src/logging.ts +44 -0
- package/src/mcp/server.ts +126 -0
- package/src/progress.ts +28 -0
- package/src/retrieval/bm25.ts +22 -0
- package/src/retrieval/reranker.ts +64 -0
- package/src/retrieval/search.ts +144 -0
- package/src/security.ts +46 -0
- package/src/storage/fts.ts +130 -0
- package/src/storage/metadata.ts +162 -0
- package/src/vector/lancedb.ts +95 -0
- package/src/vector/store.ts +8 -0
- package/src/version.ts +55 -0
- package/src/watcher/index.ts +77 -0
- package/tsconfig.json +21 -0
package/.edhindexignore
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
node_modules
|
|
2
|
+
.git
|
|
3
|
+
dist
|
|
4
|
+
build
|
|
5
|
+
coverage
|
|
6
|
+
.next
|
|
7
|
+
out
|
|
8
|
+
vendor
|
|
9
|
+
bin
|
|
10
|
+
obj
|
|
11
|
+
target
|
|
12
|
+
tmp
|
|
13
|
+
.cache
|
|
14
|
+
.turbo
|
|
15
|
+
.edhindex
|
|
16
|
+
*.exe
|
|
17
|
+
*.dll
|
|
18
|
+
*.so
|
|
19
|
+
*.dylib
|
|
20
|
+
*.bin
|
|
21
|
+
*.wasm
|
|
22
|
+
*.o
|
|
23
|
+
*.a
|
|
24
|
+
*.lib
|
|
25
|
+
*.pyc
|
|
26
|
+
*.class
|
|
27
|
+
*.jar
|
|
28
|
+
*.war
|
|
29
|
+
*.png
|
|
30
|
+
*.jpg
|
|
31
|
+
*.jpeg
|
|
32
|
+
*.gif
|
|
33
|
+
*.svg
|
|
34
|
+
*.ico
|
|
35
|
+
*.webp
|
|
36
|
+
*.bmp
|
|
37
|
+
*.tiff
|
|
38
|
+
*.ico
|
|
39
|
+
*.pdf
|
|
40
|
+
*.doc
|
|
41
|
+
*.docx
|
|
42
|
+
*.xls
|
|
43
|
+
*.xlsx
|
|
44
|
+
*.ppt
|
|
45
|
+
*.pptx
|
|
46
|
+
*.zip
|
|
47
|
+
*.tar
|
|
48
|
+
*.gz
|
|
49
|
+
*.bz2
|
|
50
|
+
*.7z
|
|
51
|
+
*.rar
|
|
52
|
+
*.mp4
|
|
53
|
+
*.avi
|
|
54
|
+
*.mov
|
|
55
|
+
*.wmv
|
|
56
|
+
*.flv
|
|
57
|
+
*.mkv
|
|
58
|
+
*.mp3
|
|
59
|
+
*.wav
|
|
60
|
+
*.flac
|
|
61
|
+
*.ogg
|
|
62
|
+
*.min.js
|
|
63
|
+
*.min.css
|
|
64
|
+
package-lock.json
|
|
65
|
+
yarn.lock
|
|
66
|
+
pnpm-lock.yaml
|
package/BUILD_BRIEF.md
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# EDHIndex — build brief
|
|
2
|
+
|
|
3
|
+
## How to use this
|
|
4
|
+
Save this file as `BUILD_BRIEF.md` in the repo root. Works with whatever AI coding agent you're using — GitHub Copilot, Claude Code, Cursor, Windsurf, or anything else. Reference it however that tool supports. Build the v1 section fully, end to end, before opening the v2 section — v2 exists so an eager agent doesn't try to build everything at once.
|
|
5
|
+
|
|
6
|
+
## What this is
|
|
7
|
+
EDHIndex is a standalone local tool that indexes a codebase and exposes fast, hybrid code search (keyword + semantic + reranked) as an MCP server. Any MCP-compatible client can connect and call it. Self-contained — no dependency on any editor, terminal, or agent application.
|
|
8
|
+
|
|
9
|
+
## Think of it as a code intelligence engine, not "a RAG project"
|
|
10
|
+
RAG is one consumer of this engine, not the whole identity of it. Layers, top to bottom:
|
|
11
|
+
```
|
|
12
|
+
CLI
|
|
13
|
+
|
|
|
14
|
+
Configuration
|
|
15
|
+
|
|
|
16
|
+
Workspace manager
|
|
17
|
+
|
|
|
18
|
+
Indexer (tree-sitter parser, file watcher, git integration)
|
|
19
|
+
|
|
|
20
|
+
Storage (SQLite: metadata + FTS5 | LanceDB: embeddings)
|
|
21
|
+
|
|
|
22
|
+
Retrieval engine (BM25, vector search, rerank)
|
|
23
|
+
|
|
|
24
|
+
MCP server
|
|
25
|
+
```
|
|
26
|
+
This separation is what lets you later build a CLI grep tool, a VS Code extension, or anything else on the same core without touching the indexing or retrieval layers.
|
|
27
|
+
|
|
28
|
+
## Naming conventions
|
|
29
|
+
- Repo / npm package / CLI command: `edhindex` — no exact match found on npm at time of writing, but double-check on npmjs.com right before first publish since registry search results can lag.
|
|
30
|
+
- Local index folder inside any indexed project: `.edhindex/`
|
|
31
|
+
- Ignore file: `.edhindexignore` (same syntax as `.gitignore`)
|
|
32
|
+
|
|
33
|
+
## Design principles
|
|
34
|
+
These are the non-negotiable values the rest of this brief is derived from. When a build decision isn't spelled out explicitly elsewhere, default back to these:
|
|
35
|
+
- Local-first — the tool works fully on the user's machine
|
|
36
|
+
- Offline-first — no network dependency after the embedding model is downloaded once
|
|
37
|
+
- Zero telemetry
|
|
38
|
+
- No vendor lock-in — interfaces over concrete libraries, everywhere it matters
|
|
39
|
+
- Deterministic indexing — same input produces the same index
|
|
40
|
+
- Incremental updates only — never re-do work that isn't stale
|
|
41
|
+
- Safe-by-default — security guardrails are load-bearing, not optional
|
|
42
|
+
- Extensible without rewriting core components
|
|
43
|
+
|
|
44
|
+
## Distribution model — git repo, not a hosted service
|
|
45
|
+
- User runs `git clone`, then `npm install`
|
|
46
|
+
- `npx edhindex start` launches it locally, pointed at whichever folder the user opens
|
|
47
|
+
- Everything happens inside that folder — nothing hosted, nothing leaves the machine
|
|
48
|
+
|
|
49
|
+
## Model tiers — user picks one on first run
|
|
50
|
+
- **Light** — bge-small-en-v1.5 (~130MB) or all-MiniLM-L6-v2 (~46MB). Old/weak hardware.
|
|
51
|
+
- **Balanced** (default) — nomic-embed-text-v1.5/v2 (~270MB). 8,192 token context, best size-to-quality tradeoff.
|
|
52
|
+
- **Max quality** — BGE-M3 (~1.1-2.2GB). Matches commercial APIs on benchmarks, needs more RAM/CPU.
|
|
53
|
+
|
|
54
|
+
All run locally via `@huggingface/transformers` — no API call, no account, no ongoing cost at any tier. Switching tiers later requires a full re-index — vectors from different models aren't compatible with each other.
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
# V1 — build this first, ship it, then stop
|
|
59
|
+
|
|
60
|
+
## Supported languages (v1)
|
|
61
|
+
TypeScript, JavaScript, Go, Python. Nothing else yet — don't let an agent try to wire up every Tree-sitter grammar that exists.
|
|
62
|
+
|
|
63
|
+
## Never index these (v1)
|
|
64
|
+
`node_modules`, `.git`, `dist`, `build`, `coverage`, `.next`, `out`, `vendor`, `bin`, `obj`, `target`, `tmp`, `.cache`, `.turbo` — plus whatever the user adds to `.edhindexignore`.
|
|
65
|
+
|
|
66
|
+
## File limits (v1)
|
|
67
|
+
Skip anything over 10MB. Skip binaries, images, PDFs, videos, archives, and anything that looks generated.
|
|
68
|
+
|
|
69
|
+
## Chunking strategy (v1)
|
|
70
|
+
Priority order: function → method → class → interface → enum → module → file. If a unit exceeds the embedding model's token limit, split it recursively rather than truncating silently.
|
|
71
|
+
|
|
72
|
+
## Metadata schema (v1)
|
|
73
|
+
Store per chunk in SQLite: `id, file, language, symbol, kind, parent, hash, git_commit, start_line, end_line, imports, exports, embedding_model, last_indexed, checksum`. Define this once, up front — retrofitting new fields into an existing index later is painful.
|
|
74
|
+
|
|
75
|
+
## Index versioning (v1)
|
|
76
|
+
`.edhindex/version.json`, written on every index build:
|
|
77
|
+
```json
|
|
78
|
+
{
|
|
79
|
+
"schema": 1,
|
|
80
|
+
"embeddingModel": "nomic-v2",
|
|
81
|
+
"chunkStrategy": 1,
|
|
82
|
+
"createdAt": "2026-07-19T00:00:00Z",
|
|
83
|
+
"lastIndexed": "2026-07-19T00:00:00Z"
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
On startup, compare this against the current build's expected values. Any mismatch (schema bump, model change, chunk strategy change) → trigger a full rebuild automatically rather than surfacing a confusing runtime error. This is the mechanism that makes "switching tiers requires a full re-index" (above) something the tool detects and handles, not something the user has to remember.
|
|
87
|
+
|
|
88
|
+
## Incremental indexing (v1)
|
|
89
|
+
Hash each file (SHA256). Chokidar triggers a check on save; only re-parse and re-embed if the hash actually changed. Don't re-embed unchanged files just because a save event fired.
|
|
90
|
+
Also hash each chunk individually. When a file's hash changes, diff at the chunk level and only re-embed the chunks whose hash actually changed — not the whole file. A one-function edit in a 2,000-line file should cost one embedding call, not fifty.
|
|
91
|
+
|
|
92
|
+
## Startup flow (v1)
|
|
93
|
+
- No index found → full scan → build → ready
|
|
94
|
+
- Existing index found → load it → compare hashes → update only what changed → ready
|
|
95
|
+
|
|
96
|
+
## Ranking formula (v1)
|
|
97
|
+
Top 30 BM25 candidates + top 30 vector candidates → deduplicate → rerank the merged set → return top 10 (configurable). Concrete numbers, not "combine and rerank" left vague. Full pipeline, in order:
|
|
98
|
+
```
|
|
99
|
+
Query
|
|
100
|
+
|
|
|
101
|
+
Normalize
|
|
102
|
+
|
|
|
103
|
+
BM25 search (top 30) + Vector search (top 30)
|
|
104
|
+
|
|
|
105
|
+
Merge
|
|
106
|
+
|
|
|
107
|
+
Deduplicate
|
|
108
|
+
|
|
|
109
|
+
Rerank
|
|
110
|
+
|
|
|
111
|
+
Top K (default 10)
|
|
112
|
+
|
|
|
113
|
+
MCP response
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Embedding and storage abstraction (v1 — cheap now, expensive to retrofit later)
|
|
117
|
+
Define interfaces, don't hardcode the concrete libraries:
|
|
118
|
+
```
|
|
119
|
+
EmbeddingProvider: embed(), dimensions(), name(), load(), dispose()
|
|
120
|
+
VectorStore: insert(), delete(), search(), compact()
|
|
121
|
+
```
|
|
122
|
+
Ship transformers.js and LanceDB as the default implementations of these interfaces. This is what lets you swap to Ollama, llama.cpp, Qdrant, or anything else later without rewriting the code that calls them.
|
|
123
|
+
|
|
124
|
+
## Config file (v1)
|
|
125
|
+
`.edhindex/config.json`:
|
|
126
|
+
```json
|
|
127
|
+
{
|
|
128
|
+
"model": "balanced",
|
|
129
|
+
"languages": ["ts", "js", "go", "py"],
|
|
130
|
+
"watch": true,
|
|
131
|
+
"rerank": true,
|
|
132
|
+
"maxResults": 10
|
|
133
|
+
}
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## Progress reporting (v1)
|
|
137
|
+
Indexing must never look hung. Emit structured progress events (not just log lines) that a CLI progress bar or an MCP client can both consume — e.g. `{ phase: "scanning", current: 118, total: 624 }`, then `{ phase: "embedding", percent: 42 }`, then `{ phase: "building_fts" }`, then `{ phase: "ready" }`.
|
|
138
|
+
|
|
139
|
+
## Cancellation (v1)
|
|
140
|
+
Indexing must be safely interruptible (Ctrl+C, or an MCP-triggered stop). On interrupt: finish the file currently being processed, flush metadata to SQLite, close the DB cleanly, then exit. Never leave a half-written index — a killed process should produce either the old valid state or the new valid state, never something in between.
|
|
141
|
+
|
|
142
|
+
## Logging levels (v1)
|
|
143
|
+
Four levels, fixed from the start so an agent doesn't scatter ad hoc `console.log` calls: `silent`, `info` (default), `verbose`, `debug`. Configurable via CLI flag and config file.
|
|
144
|
+
|
|
145
|
+
## CLI (v1)
|
|
146
|
+
```
|
|
147
|
+
edhindex init
|
|
148
|
+
edhindex start
|
|
149
|
+
edhindex index
|
|
150
|
+
edhindex search <query>
|
|
151
|
+
edhindex status
|
|
152
|
+
edhindex config
|
|
153
|
+
edhindex models
|
|
154
|
+
edhindex doctor
|
|
155
|
+
```
|
|
156
|
+
`doctor` checks SQLite, LanceDB, the embedding model, the MCP server, disk space, permissions, and active file watchers. Build this in v1 — it pays for itself the very first time something breaks and you need to know where.
|
|
157
|
+
|
|
158
|
+
## Security — non-negotiable, build this in from the start, not after
|
|
159
|
+
- Never execute any indexed code
|
|
160
|
+
- Never follow symlinks that point outside the selected workspace
|
|
161
|
+
- Never transmit source code over the network — enforce this in code, not just as a policy, unless the user explicitly opts into a future remote embedding provider
|
|
162
|
+
- Sanitize every path to prevent directory traversal
|
|
163
|
+
- Restrict every MCP tool operation strictly to the indexed workspace — a calling agent should never be able to reach outside the folder it was pointed at
|
|
164
|
+
|
|
165
|
+
This matters because once this runs as an MCP server, it can be driven by an autonomous agent acting on file contents it reads — path traversal and symlink escapes are the first thing that will get tried against a tool like this, intentionally or not.
|
|
166
|
+
|
|
167
|
+
## MCP tools exposed (v1)
|
|
168
|
+
`search_codebase` only. Everything else is v2.
|
|
169
|
+
|
|
170
|
+
## Non-goals (v1)
|
|
171
|
+
Explicit so an eager agent doesn't add any of these on its own initiative:
|
|
172
|
+
- No chat interface
|
|
173
|
+
- No code generation
|
|
174
|
+
- No cloud sync
|
|
175
|
+
- No telemetry
|
|
176
|
+
- No authentication
|
|
177
|
+
- No web UI
|
|
178
|
+
- No editor integration
|
|
179
|
+
- No remote database
|
|
180
|
+
- No hosted service
|
|
181
|
+
|
|
182
|
+
## Corrections worth keeping in mind
|
|
183
|
+
- MCP was created by Anthropic (Nov 2024), not OpenAI — now an open, vendor-neutral standard under the Linux Foundation. Use `@modelcontextprotocol/sdk` or FastMCP.
|
|
184
|
+
- `tiktoken` matches OpenAI's tokenizer specifically — treat any token counts here as approximate for other models.
|
|
185
|
+
|
|
186
|
+
## Recommended stack (v1)
|
|
187
|
+
Tree-sitter (AST), SQLite (metadata + FTS5), LanceDB (vectors), transformers.js (embeddings + reranker, local), chokidar (file watching), simple-git (git integration), MCP server (FastMCP or TypeScript SDK).
|
|
188
|
+
|
|
189
|
+
Note: Language Server Protocol integration is intentionally **not** in v1. Tree-sitter alone (function/class/import/export boundaries) is enough for a strong index and search experience. LSP adds per-language server lifecycle management, installation, and cross-platform quirks — real complexity that isn't needed until the persisted symbol/call graph (v2) makes it worth it. Don't let an agent reach for LSP in v1.
|
|
190
|
+
|
|
191
|
+
## Build order (v1)
|
|
192
|
+
1. Repo scaffold, config file, CLI skeleton, ignore rules, file size limits
|
|
193
|
+
2. AST indexing — Tree-sitter, chunking strategy, metadata schema, incremental hashing
|
|
194
|
+
3. Embeddings and vector search — via the `EmbeddingProvider` interface, tier selection on first run
|
|
195
|
+
4. Keyword search — SQLite FTS5
|
|
196
|
+
5. Ranking — the top-30/top-30/dedupe/rerank/top-10 formula
|
|
197
|
+
6. MCP server exposing `search_codebase`, with the security guardrails enforced here
|
|
198
|
+
7. File watching and the cold-start/warm-start flow
|
|
199
|
+
8. `doctor` command and basic recovery (if SQLite fails to open, rebuild from scratch rather than crash)
|
|
200
|
+
|
|
201
|
+
## Success criteria (v1)
|
|
202
|
+
v1 is done when all of the following hold:
|
|
203
|
+
- Indexes a 100k-line repository successfully
|
|
204
|
+
- Updates incrementally on file save, without a full re-index
|
|
205
|
+
- Returns search results in a reasonable time on a warm index
|
|
206
|
+
- Runs completely offline after the embedding model is downloaded
|
|
207
|
+
- Emits zero network requests beyond that one-time model download
|
|
208
|
+
- Recovers cleanly if `.edhindex/` is deleted (falls back to full scan)
|
|
209
|
+
- Runs on Windows, macOS, and Linux
|
|
210
|
+
- `search_codebase` works end to end through an MCP client
|
|
211
|
+
|
|
212
|
+
(Hard performance numbers — search latency, index update time, cold start, memory ceiling — are deliberately a v2 tuning target, not a v1 gate. Don't block shipping v1 on numbers that haven't been measured yet.)
|
|
213
|
+
|
|
214
|
+
---
|
|
215
|
+
|
|
216
|
+
# V2 — deliberately deferred, don't touch until v1 works end to end
|
|
217
|
+
|
|
218
|
+
- **Persisted symbol/call graph** (LSP-derived) — enables "expand from a chunk to its related symbols" as part of retrieval. This is the single biggest quality upgrade available, and also the most complex — earn it after the simpler retrieval loop is proven, not before.
|
|
219
|
+
- **Additional MCP tools**: `find_symbol`, `find_references`, `find_definition`, `recent_changes`, `workspace_stats`, `reindex`, `index_status`
|
|
220
|
+
- **Parallel indexing via worker threads** — only once you've actually confirmed indexing speed is a bottleneck on a real repo, not preemptively
|
|
221
|
+
- **Additional languages** beyond the v1 four (Java, C#, Rust, C++, Kotlin, Swift)
|
|
222
|
+
- **Formal test suite** as its own tracked effort — unit, integration, large-repo, corrupt-index, model-switch, recovery, watcher tests
|
|
223
|
+
- **Sophisticated corruption recovery** — attempt repair before falling back to a full rebuild
|
|
224
|
+
- **Full plugin system** — v1 already reserves the extension points via the two interfaces above; a real plugin loader can wait
|
|
225
|
+
- **Performance target tuning** — search under 300ms, index update under 2s, cold startup under 10s, memory under 1GB. Good targets to measure against once v1 exists, not a blocker before it ships
|
|
226
|
+
|
|
227
|
+
## Scope check
|
|
228
|
+
This is the entire scope of EDHIndex: index a codebase, keep the index live, expose it over MCP, safely. It does not include an editor, a terminal, or a chat interface — those are separate projects, if built at all.
|
package/package.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "edhindex",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Local-first hybrid code search engine with MCP",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"edhindex": "./dist/cli/index.js"
|
|
8
|
+
},
|
|
9
|
+
"exports": {
|
|
10
|
+
".": "./dist/index.js"
|
|
11
|
+
},
|
|
12
|
+
"scripts": {
|
|
13
|
+
"build": "tsc",
|
|
14
|
+
"start": "node dist/cli/index.js",
|
|
15
|
+
"dev": "npx tsx src/cli/index.ts"
|
|
16
|
+
},
|
|
17
|
+
"dependencies": {
|
|
18
|
+
"@huggingface/transformers": "^3.8.1",
|
|
19
|
+
"@lancedb/lancedb": "^0.31.0",
|
|
20
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
21
|
+
"better-sqlite3": "^12.11.1",
|
|
22
|
+
"chalk": "^5.6.2",
|
|
23
|
+
"chokidar": "^5.0.0",
|
|
24
|
+
"cli-progress": "^3.12.0",
|
|
25
|
+
"commander": "^15.0.0",
|
|
26
|
+
"simple-git": "^3.36.0",
|
|
27
|
+
"tree-sitter-go": "^0.25.0",
|
|
28
|
+
"tree-sitter-python": "^0.25.0",
|
|
29
|
+
"tree-sitter-typescript": "^0.23.2",
|
|
30
|
+
"web-tree-sitter": "^0.26.11"
|
|
31
|
+
},
|
|
32
|
+
"devDependencies": {
|
|
33
|
+
"@types/better-sqlite3": "^7.6.13",
|
|
34
|
+
"@types/cli-progress": "^3.11.6",
|
|
35
|
+
"@types/node": "^22.13.0",
|
|
36
|
+
"typescript": "^5.7.0"
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { loadConfig, saveConfig, ModelTier, LogLevelConfig } from '../../config/index.js';
|
|
4
|
+
import chalk from 'chalk';
|
|
5
|
+
|
|
6
|
+
export function configCommand(rootPath: string, changes: Record<string, string>) {
|
|
7
|
+
const indexDir = join(rootPath, '.edhindex');
|
|
8
|
+
|
|
9
|
+
if (!existsSync(indexDir)) {
|
|
10
|
+
console.log(chalk.yellow('No .edhindex directory found. Run "edhindex init" first.'));
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const config = loadConfig(indexDir);
|
|
15
|
+
|
|
16
|
+
if (Object.keys(changes).length === 0) {
|
|
17
|
+
console.log(chalk.bold('Current configuration:'));
|
|
18
|
+
console.log(JSON.stringify(config, null, 2));
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
for (const [key, value] of Object.entries(changes)) {
|
|
23
|
+
switch (key) {
|
|
24
|
+
case 'model':
|
|
25
|
+
if (!['light', 'balanced', 'max'].includes(value)) {
|
|
26
|
+
console.log(chalk.red(`Invalid model: ${value}. Must be light, balanced, or max.`));
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
(config as any).model = value as ModelTier;
|
|
30
|
+
break;
|
|
31
|
+
case 'languages':
|
|
32
|
+
config.languages = value.split(',').map(s => s.trim());
|
|
33
|
+
break;
|
|
34
|
+
case 'watch':
|
|
35
|
+
config.watch = value === 'true';
|
|
36
|
+
break;
|
|
37
|
+
case 'rerank':
|
|
38
|
+
config.rerank = value === 'true';
|
|
39
|
+
break;
|
|
40
|
+
case 'maxResults':
|
|
41
|
+
config.maxResults = parseInt(value, 10) || 10;
|
|
42
|
+
break;
|
|
43
|
+
case 'logLevel':
|
|
44
|
+
if (!['silent', 'info', 'verbose', 'debug'].includes(value)) {
|
|
45
|
+
console.log(chalk.red(`Invalid logLevel: ${value}`));
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
config.logLevel = value as LogLevelConfig;
|
|
49
|
+
break;
|
|
50
|
+
default:
|
|
51
|
+
console.log(chalk.yellow(`Unknown config key: ${key}`));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
saveConfig(indexDir, config);
|
|
56
|
+
console.log(chalk.green('Configuration updated.'));
|
|
57
|
+
console.log(JSON.stringify(config, null, 2));
|
|
58
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { runDoctor, DoctorResult } from '../../doctor/index.js';
|
|
2
|
+
import chalk from 'chalk';
|
|
3
|
+
|
|
4
|
+
export async function doctorCommand(rootPath: string, indexDir: string) {
|
|
5
|
+
console.log(chalk.bold('EDHIndex Diagnostics\n'));
|
|
6
|
+
|
|
7
|
+
const results = await runDoctor(rootPath, indexDir);
|
|
8
|
+
|
|
9
|
+
let okCount = 0;
|
|
10
|
+
let warnCount = 0;
|
|
11
|
+
let errorCount = 0;
|
|
12
|
+
|
|
13
|
+
for (const r of results) {
|
|
14
|
+
const icon = r.status === 'ok' ? chalk.green('✓') : r.status === 'warn' ? chalk.yellow('⚠') : chalk.red('✗');
|
|
15
|
+
console.log(` ${icon} ${chalk.bold(r.category)}: ${r.message}`);
|
|
16
|
+
if (r.detail) console.log(` ${chalk.dim(r.detail)}`);
|
|
17
|
+
|
|
18
|
+
if (r.status === 'ok') okCount++;
|
|
19
|
+
else if (r.status === 'warn') warnCount++;
|
|
20
|
+
else errorCount++;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
console.log('');
|
|
24
|
+
if (errorCount > 0) {
|
|
25
|
+
console.log(chalk.red(` ${errorCount} error(s), ${warnCount} warning(s), ${okCount} ok`));
|
|
26
|
+
} else if (warnCount > 0) {
|
|
27
|
+
console.log(chalk.yellow(` ${warnCount} warning(s), ${okCount} ok`));
|
|
28
|
+
} else {
|
|
29
|
+
console.log(chalk.green(` All ${okCount} checks passed.`));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { existsSync, mkdirSync } from 'node:fs';
|
|
3
|
+
import { loadConfig, saveConfig, MODEL_CONFIGS } from '../../config/index.js';
|
|
4
|
+
import { logger, setLogLevel, LogLevel } from '../../logging.js';
|
|
5
|
+
import { emitProgress, onProgress } from '../../progress.js';
|
|
6
|
+
import { loadVersion, saveVersion, getExpectedVersion, needsRebuild } from '../../version.js';
|
|
7
|
+
import { MetadataStore } from '../../storage/metadata.js';
|
|
8
|
+
import { FTSStore } from '../../storage/fts.js';
|
|
9
|
+
import { TransformersEmbeddingProvider } from '../../embeddings/transformers.js';
|
|
10
|
+
import { LanceDBStore } from '../../vector/lancedb.js';
|
|
11
|
+
import { Indexer } from '../../indexer/indexer.js';
|
|
12
|
+
import cliProgress from 'cli-progress';
|
|
13
|
+
import chalk from 'chalk';
|
|
14
|
+
|
|
15
|
+
export async function indexCommand(rootPath: string) {
|
|
16
|
+
const indexDir = join(rootPath, '.edhindex');
|
|
17
|
+
if (!existsSync(indexDir)) {
|
|
18
|
+
mkdirSync(indexDir, { recursive: true });
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const config = loadConfig(indexDir);
|
|
22
|
+
const modelCfg = MODEL_CONFIGS[config.model];
|
|
23
|
+
|
|
24
|
+
setLogLevel(LogLevel.Info);
|
|
25
|
+
|
|
26
|
+
// Check version
|
|
27
|
+
const version = loadVersion(indexDir);
|
|
28
|
+
const fullRebuild = needsRebuild(version, config.model);
|
|
29
|
+
|
|
30
|
+
if (fullRebuild && version) {
|
|
31
|
+
console.log(chalk.yellow('Index version mismatch — full rebuild.'));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const metadataStore = new MetadataStore(indexDir);
|
|
35
|
+
const ftsStore = new FTSStore(indexDir);
|
|
36
|
+
const vectorStore = new LanceDBStore(indexDir, modelCfg.dimensions);
|
|
37
|
+
await vectorStore.init();
|
|
38
|
+
|
|
39
|
+
const embedder = new TransformersEmbeddingProvider(config.model);
|
|
40
|
+
console.log(chalk.dim('Loading embedding model...'));
|
|
41
|
+
await embedder.load();
|
|
42
|
+
|
|
43
|
+
const indexer = new Indexer({
|
|
44
|
+
config,
|
|
45
|
+
rootPath,
|
|
46
|
+
indexDir,
|
|
47
|
+
onIndex: async (result) => {
|
|
48
|
+
for (const chunk of result.chunks) {
|
|
49
|
+
metadataStore.upsertChunk({
|
|
50
|
+
id: chunk.id, file: chunk.file, language: chunk.language,
|
|
51
|
+
symbol: chunk.symbol, kind: chunk.kind, parent: chunk.parent,
|
|
52
|
+
hash: chunk.hash, git_commit: null,
|
|
53
|
+
start_line: chunk.startLine, end_line: chunk.endLine,
|
|
54
|
+
imports: chunk.imports.join('\n'), exports: chunk.exports.join('\n'),
|
|
55
|
+
embedding_model: embedder.name(), last_indexed: new Date().toISOString(),
|
|
56
|
+
checksum: chunk.hash,
|
|
57
|
+
});
|
|
58
|
+
ftsStore.insertChunk(chunk.id, chunk.content, chunk.file, chunk.symbol, chunk.language);
|
|
59
|
+
}
|
|
60
|
+
if (result.chunks.length > 0) {
|
|
61
|
+
const texts = result.chunks.map(c => c.content);
|
|
62
|
+
const embeddings = await embedder.embed(texts);
|
|
63
|
+
await vectorStore.insert(result.chunks.map((c, i) => ({
|
|
64
|
+
id: c.id, values: embeddings[i],
|
|
65
|
+
metadata: { file: c.file, symbol: c.symbol, kind: c.kind, text: c.content, language: c.language },
|
|
66
|
+
})));
|
|
67
|
+
}
|
|
68
|
+
metadataStore.upsertFileHash(result.relativePath, result.hash);
|
|
69
|
+
},
|
|
70
|
+
onDelete: async (chunkIds) => {
|
|
71
|
+
for (const id of chunkIds) ftsStore.deleteChunk(id);
|
|
72
|
+
await vectorStore.delete(chunkIds);
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
await indexer.initialize();
|
|
77
|
+
|
|
78
|
+
const progressBar = new cliProgress.SingleBar({
|
|
79
|
+
format: '{phase} | {bar} | {percentage}% | {value}/{total}',
|
|
80
|
+
barCompleteChar: '\u2588',
|
|
81
|
+
barIncompleteChar: '\u2591',
|
|
82
|
+
hideCursor: true,
|
|
83
|
+
});
|
|
84
|
+
progressBar.start(100, 0, { phase: 'Scanning' });
|
|
85
|
+
const unsub = onProgress((e) => {
|
|
86
|
+
if (e.phase === 'scanning' && e.total) {
|
|
87
|
+
progressBar.setTotal(e.total);
|
|
88
|
+
progressBar.update(e.current || 0);
|
|
89
|
+
} else if (e.phase === 'embedding' && e.percent !== undefined) {
|
|
90
|
+
progressBar.setTotal(100);
|
|
91
|
+
progressBar.update(e.percent);
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
const existingHashes = metadataStore.getAllFileHashes();
|
|
96
|
+
const doFull = fullRebuild || existingHashes.size === 0;
|
|
97
|
+
|
|
98
|
+
if (doFull) {
|
|
99
|
+
await indexer.runFullIndex();
|
|
100
|
+
const newVersion = getExpectedVersion(config.model);
|
|
101
|
+
saveVersion(indexDir, newVersion);
|
|
102
|
+
} else {
|
|
103
|
+
const existingChunkHashes = metadataStore.getAllChunkHashes();
|
|
104
|
+
await indexer.runIncremental(existingHashes, existingChunkHashes);
|
|
105
|
+
const ver = version!;
|
|
106
|
+
ver.lastIndexed = new Date().toISOString();
|
|
107
|
+
saveVersion(indexDir, ver);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
progressBar.stop();
|
|
111
|
+
unsub();
|
|
112
|
+
|
|
113
|
+
const finalChunkCount = metadataStore.getChunkCount();
|
|
114
|
+
const finalFileCount = metadataStore.getFileCount();
|
|
115
|
+
console.log(chalk.green(`Index complete. ${finalChunkCount} chunks across ${finalFileCount} files.`));
|
|
116
|
+
console.log(chalk.dim(`Run "edhindex start" to serve the MCP server.`));
|
|
117
|
+
|
|
118
|
+
metadataStore.close();
|
|
119
|
+
ftsStore.close();
|
|
120
|
+
await vectorStore.close();
|
|
121
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { existsSync, mkdirSync } from 'node:fs';
|
|
3
|
+
import { loadConfig, saveConfig, defaultConfig } from '../../config/index.js';
|
|
4
|
+
import { logger } from '../../logging.js';
|
|
5
|
+
import chalk from 'chalk';
|
|
6
|
+
|
|
7
|
+
export async function initCommand(rootPath: string) {
|
|
8
|
+
const indexDir = join(rootPath, '.edhindex');
|
|
9
|
+
|
|
10
|
+
if (existsSync(indexDir)) {
|
|
11
|
+
console.log(chalk.yellow('EDHIndex is already initialized in this directory.'));
|
|
12
|
+
const config = loadConfig(indexDir);
|
|
13
|
+
console.log(`Current config: model=${config.model}, languages=[${config.languages.join(', ')}]`);
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
mkdirSync(indexDir, { recursive: true });
|
|
18
|
+
const config = defaultConfig();
|
|
19
|
+
saveConfig(indexDir, config);
|
|
20
|
+
|
|
21
|
+
console.log(chalk.green('EDHIndex initialized.'));
|
|
22
|
+
console.log(`Index directory: ${indexDir}`);
|
|
23
|
+
console.log(`Configuration: model=${config.model}, languages=[${config.languages.join(', ')}]`);
|
|
24
|
+
console.log(chalk.dim('Run "edhindex start" to build the index and start the MCP server.'));
|
|
25
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { MODEL_CONFIGS } from '../../config/index.js';
|
|
2
|
+
import chalk from 'chalk';
|
|
3
|
+
|
|
4
|
+
export function modelsCommand() {
|
|
5
|
+
console.log(chalk.bold('Available embedding models:\n'));
|
|
6
|
+
|
|
7
|
+
for (const [tier, cfg] of Object.entries(MODEL_CONFIGS)) {
|
|
8
|
+
const tag = tier === 'balanced' ? ' (default)' : '';
|
|
9
|
+
console.log(` ${chalk.cyan(tier)}${chalk.dim(tag)}`);
|
|
10
|
+
console.log(` Model: ${cfg.displayName}`);
|
|
11
|
+
console.log(` Size: ${cfg.size}`);
|
|
12
|
+
console.log(` Dims: ${cfg.dimensions}`);
|
|
13
|
+
console.log(` HF ID: ${cfg.modelId}`);
|
|
14
|
+
console.log('');
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
console.log(chalk.dim('Switch models with: edhindex config model <tier>'));
|
|
18
|
+
console.log(chalk.dim('Note: Changing models requires a full re-index.'));
|
|
19
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { loadConfig, MODEL_CONFIGS } from '../../config/index.js';
|
|
3
|
+
import { setLogLevel, LogLevel } from '../../logging.js';
|
|
4
|
+
import { MetadataStore } from '../../storage/metadata.js';
|
|
5
|
+
import { FTSStore } from '../../storage/fts.js';
|
|
6
|
+
import { TransformersEmbeddingProvider } from '../../embeddings/transformers.js';
|
|
7
|
+
import { LanceDBStore } from '../../vector/lancedb.js';
|
|
8
|
+
import { SearchEngine } from '../../retrieval/search.js';
|
|
9
|
+
import chalk from 'chalk';
|
|
10
|
+
|
|
11
|
+
export async function searchCommand(rootPath: string, query: string, maxResults: number) {
|
|
12
|
+
const indexDir = join(rootPath, '.edhindex');
|
|
13
|
+
setLogLevel(LogLevel.Silent);
|
|
14
|
+
|
|
15
|
+
const config = loadConfig(indexDir);
|
|
16
|
+
const modelCfg = MODEL_CONFIGS[config.model];
|
|
17
|
+
|
|
18
|
+
const metadataStore = new MetadataStore(indexDir);
|
|
19
|
+
const ftsStore = new FTSStore(indexDir);
|
|
20
|
+
const vectorStore = new LanceDBStore(indexDir, modelCfg.dimensions);
|
|
21
|
+
await vectorStore.init();
|
|
22
|
+
|
|
23
|
+
const embedder = new TransformersEmbeddingProvider(config.model);
|
|
24
|
+
await embedder.load();
|
|
25
|
+
|
|
26
|
+
const engine = new SearchEngine(ftsStore, metadataStore, vectorStore, embedder, null, false);
|
|
27
|
+
const results = await engine.search({ query, maxResults });
|
|
28
|
+
|
|
29
|
+
if (results.length === 0) {
|
|
30
|
+
console.log(chalk.yellow('No results found.'));
|
|
31
|
+
} else {
|
|
32
|
+
for (const r of results) {
|
|
33
|
+
const tag = r.matchType === 'keyword' ? chalk.blue('KW') : r.matchType === 'vector' ? chalk.magenta('VEC') : chalk.green('HYB');
|
|
34
|
+
console.log(`${tag} ${chalk.cyan(`${r.file}:${r.startLine}-${r.endLine}`)} (${r.language})`);
|
|
35
|
+
if (r.symbol) console.log(` ${chalk.dim('Symbol:')} ${r.symbol}${r.kind ? ` (${r.kind})` : ''}`);
|
|
36
|
+
console.log(` ${chalk.dim('Score:')} ${r.score.toFixed(4)}`);
|
|
37
|
+
console.log('');
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
metadataStore.close();
|
|
42
|
+
ftsStore.close();
|
|
43
|
+
await vectorStore.close();
|
|
44
|
+
}
|