codesearch-mcp 0.2.2-alpha → 0.3.0-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -1
- package/README.md +60 -33
- package/dist/app.js +46 -22
- package/dist/src/chunking/AstChunker.d.ts +12 -0
- package/dist/src/chunking/AstChunker.js +30 -0
- package/dist/src/chunking/CharacterChunkStrategy.d.ts +8 -0
- package/dist/src/chunking/CharacterChunkStrategy.js +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.d.ts +15 -0
- package/dist/src/chunking/DeclarationChunkStrategy.js +87 -0
- package/dist/src/chunking/FragmentMerger.d.ts +11 -0
- package/dist/src/chunking/FragmentMerger.js +61 -0
- package/dist/src/chunking/ImportOnlyFilter.d.ts +5 -0
- package/dist/src/chunking/ImportOnlyFilter.js +14 -0
- package/dist/src/chunking/LangchainTextSplitter.d.ts +21 -0
- package/dist/src/chunking/LangchainTextSplitter.js +133 -0
- package/dist/src/chunking/MinSizeSplitPolicy.d.ts +9 -0
- package/dist/src/chunking/MinSizeSplitPolicy.js +18 -0
- package/dist/src/chunking/RunPacker.d.ts +12 -0
- package/dist/src/chunking/RunPacker.js +84 -0
- package/dist/src/chunking/create-chunker.d.ts +3 -0
- package/dist/src/chunking/create-chunker.js +15 -0
- package/dist/src/chunking/interfaces.d.ts +27 -0
- package/dist/src/chunking/interfaces.js +2 -0
- package/dist/src/commands/ConfigCommand.js +1 -9
- package/dist/src/commands/EvalCommand.d.ts +11 -0
- package/dist/src/commands/EvalCommand.js +264 -0
- package/dist/src/commands/IndexCommand.js +70 -66
- package/dist/src/commands/McpCommand.js +2 -8
- package/dist/src/commands/WebCommand.js +2 -5
- package/dist/src/config-error.d.ts +0 -4
- package/dist/src/config-error.js +0 -4
- package/dist/src/config-store.d.ts +1 -54
- package/dist/src/config-store.js +8 -69
- package/dist/src/config.d.ts +8 -17
- package/dist/src/config.js +35 -34
- package/dist/src/constants.d.ts +2 -28
- package/dist/src/constants.js +6 -28
- package/dist/src/container.d.ts +6 -0
- package/dist/src/container.js +12 -3
- package/dist/src/db/index.d.ts +1 -1
- package/dist/src/db/index.js +1 -2
- package/dist/src/db/schema.d.ts +22 -29
- package/dist/src/db/schema.js +3 -14
- package/dist/src/declarations/DeclarationTree.d.ts +19 -0
- package/dist/src/declarations/DeclarationTree.js +66 -0
- package/dist/src/declarations/DeclarationTreeBuilder.d.ts +22 -0
- package/dist/src/declarations/DeclarationTreeBuilder.js +179 -0
- package/dist/src/declarations/SourceLines.d.ts +37 -0
- package/dist/src/declarations/SourceLines.js +415 -0
- package/dist/src/dependency-traversal.d.ts +10 -0
- package/dist/src/dependency-traversal.js +39 -0
- package/dist/src/enriched-prompt.d.ts +0 -5
- package/dist/src/enriched-prompt.js +0 -5
- package/dist/src/error-record.d.ts +12 -0
- package/dist/src/error-record.js +248 -0
- package/dist/src/eval-corpus.d.ts +16 -0
- package/dist/src/eval-corpus.js +169 -0
- package/dist/src/eval-results.d.ts +15 -0
- package/dist/src/eval-results.js +397 -0
- package/dist/src/file-utils.d.ts +10 -0
- package/dist/src/file-utils.js +19 -1
- package/dist/src/graph.d.ts +0 -25
- package/dist/src/graph.js +0 -28
- package/dist/src/help.js +4 -1
- package/dist/src/hnsw-settings.d.ts +3 -0
- package/dist/src/hnsw-settings.js +24 -0
- package/dist/src/interfaces/IAstParser.d.ts +5 -30
- package/dist/src/interfaces/IChunker.d.ts +3 -12
- package/dist/src/interfaces/ICorpusInspector.d.ts +7 -0
- package/dist/src/interfaces/ICorpusInspector.js +2 -0
- package/dist/src/interfaces/IDatabase.d.ts +2 -36
- package/dist/src/interfaces/IEmbedder.d.ts +0 -9
- package/dist/src/interfaces/ILogger.d.ts +1 -11
- package/dist/src/interfaces/IMcpToolHandler.d.ts +0 -5
- package/dist/src/interfaces/IMonorepoResolver.d.ts +0 -22
- package/dist/src/interfaces/IProjectDiscovery.d.ts +0 -17
- package/dist/src/interfaces/IRetrievalEvaluator.d.ts +216 -0
- package/dist/src/interfaces/IRetrievalEvaluator.js +2 -0
- package/dist/src/interfaces/ISearcher.d.ts +26 -20
- package/dist/src/interfaces/index.d.ts +1 -0
- package/dist/src/interrupt.js +0 -2
- package/dist/src/owner-path.d.ts +9 -10
- package/dist/src/owner-path.js +11 -23
- package/dist/src/project-scope.d.ts +7 -0
- package/dist/src/project-scope.js +37 -0
- package/dist/src/project-selector.d.ts +1 -0
- package/dist/src/project-selector.js +13 -6
- package/dist/src/retrieval-eval.d.ts +24 -0
- package/dist/src/retrieval-eval.js +500 -0
- package/dist/src/search/result-shaping.d.ts +4 -0
- package/dist/src/search/result-shaping.js +19 -0
- package/dist/src/services/AstParserService.d.ts +28 -23
- package/dist/src/services/AstParserService.js +391 -98
- package/dist/src/services/DatabaseService.d.ts +2 -19
- package/dist/src/services/DatabaseService.js +44 -81
- package/dist/src/services/EmbedderFactory.d.ts +0 -4
- package/dist/src/services/EmbedderFactory.js +0 -5
- package/dist/src/services/IndexerService.d.ts +2 -10
- package/dist/src/services/IndexerService.js +29 -45
- package/dist/src/services/InfinityEmbedder.d.ts +0 -21
- package/dist/src/services/InfinityEmbedder.js +0 -15
- package/dist/src/services/LoggerService.d.ts +7 -5
- package/dist/src/services/LoggerService.js +37 -41
- package/dist/src/services/McpServer.d.ts +25 -10
- package/dist/src/services/McpServer.js +101 -64
- package/dist/src/services/MonorepoAwareDiscoveryService.d.ts +1 -45
- package/dist/src/services/MonorepoAwareDiscoveryService.js +1 -51
- package/dist/src/services/NxMonorepoResolver.d.ts +0 -29
- package/dist/src/services/NxMonorepoResolver.js +5 -56
- package/dist/src/services/OpenAICompatibleEmbedder.d.ts +0 -6
- package/dist/src/services/OpenAICompatibleEmbedder.js +1 -14
- package/dist/src/services/ProjectDiscoveryService.d.ts +0 -4
- package/dist/src/services/ProjectDiscoveryService.js +0 -4
- package/dist/src/services/SearcherService.d.ts +127 -70
- package/dist/src/services/SearcherService.js +526 -305
- package/dist/src/services/VoyageEmbedder.d.ts +0 -6
- package/dist/src/services/VoyageEmbedder.js +0 -6
- package/dist/src/services/WebServerService.d.ts +1 -53
- package/dist/src/services/WebServerService.js +11 -57
- package/dist/src/services/http-retry.d.ts +0 -7
- package/dist/src/services/http-retry.js +1 -10
- package/dist/src/services/infinity-provisioning.d.ts +0 -74
- package/dist/src/services/infinity-provisioning.js +5 -88
- package/dist/src/services/legacy-ollama-index.d.ts +0 -16
- package/dist/src/services/legacy-ollama-index.js +0 -23
- package/dist/src/services/reranker.d.ts +0 -9
- package/dist/src/services/reranker.js +46 -12
- package/dist/src/setup.d.ts +11 -23
- package/dist/src/setup.js +98 -71
- package/dist/src/symbol-name.d.ts +0 -12
- package/dist/src/symbol-name.js +0 -13
- package/dist/src/symbol-rows.d.ts +2 -11
- package/dist/src/symbol-rows.js +5 -13
- package/dist/src/tool-call-log.d.ts +2 -0
- package/dist/src/tool-call-log.js +24 -0
- package/dist/src/tools/FindSymbolTool.js +14 -5
- package/dist/src/tools/GetCallersTool.d.ts +3 -1
- package/dist/src/tools/GetCallersTool.js +16 -6
- package/dist/src/tools/GetDependenciesTool.d.ts +3 -1
- package/dist/src/tools/GetDependenciesTool.js +15 -8
- package/dist/src/tools/GetDependentsTool.d.ts +3 -1
- package/dist/src/tools/GetDependentsTool.js +31 -16
- package/dist/src/tools/ListFilesTool.d.ts +3 -1
- package/dist/src/tools/ListFilesTool.js +14 -12
- package/dist/src/tools/ListProjectsTool.js +5 -1
- package/dist/src/tools/ReadFileRangeTool.d.ts +3 -1
- package/dist/src/tools/ReadFileRangeTool.js +12 -11
- package/dist/src/tools/ReadFileTool.d.ts +3 -1
- package/dist/src/tools/ReadFileTool.js +12 -10
- package/dist/src/tools/SearchCodebaseTool.js +7 -2
- package/dist/src/tools/format-helpers.d.ts +10 -1
- package/dist/src/tools/format-helpers.js +145 -8
- package/dist/src/web/dashboard.d.ts +0 -8
- package/dist/src/web/dashboard.js +7 -10
- package/package.json +10 -9
- package/dist/src/services/AstChunker.d.ts +0 -42
- package/dist/src/services/AstChunker.js +0 -158
- package/docs/web-dashboard.md +0 -206
package/CHANGELOG.md
CHANGED
|
@@ -3,7 +3,71 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
Format based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|
5
5
|
|
|
6
|
-
## [
|
|
6
|
+
## [0.3.0-alpha] - 2026-10-31
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
|
|
10
|
+
- **One `tool call` log line per MCP tool call** — at `logging.level` `info` (or `debug`) the MCP server writes one line per completed call, whatever the outcome (result, handler error, unknown tool, refused after a failed startup, cancelled by the client): `tool`, `args` (strings cut at 500 characters, arrays at 20 items), `isError`, `responseChars` (the response size — never its content), `ms`, and the caller: `client` (`{name, version}` from `initialize`), `connectionId` (random, one per server process), `session` (`CLAUDE_CODE_SESSION_ID`, or `null`), `meta` (the request's `_meta` without `progressToken`), `callId` (a per-process counter) and, for Claude Code, `toolUseId`; `cancelled: true` marks a call the client cancelled. Find them with `jq 'select(.msg=="tool call")'`.
|
|
11
|
+
- **Call context on every MCP server log line** — every line of the MCP server carries its process's `connectionId`, and every line written during a tool call, at any level and from any service (a search debug trace, a `rerank failed` warning, a handler error), also carries that call's `callId` and `toolUseId`. `rag-index`, `rag-web` and eval logs are unchanged.
|
|
12
|
+
- **`rag-index eval` reports each search stage separately** _(internal development tool: the eval suites, corpora and results live in this repository and are not shipped in the npm package)_ — every case is searched once and scored at the `vector` stage (the candidate pool's cosine order as the reranker receives it, cut to 10 — not the same as a `rerank: false` search, which fetches a smaller pool) and, when a reranker is configured, the `reranked` stage. Each case also records where its answer sits anywhere in the candidate pool and the pool's size; the run reports pool recall and how many cases got a pool shorter than the size their search requested. Within a run, the `vector` → `reranked` change is reported per category as improved / worsened / unchanged cases, the cases that gained or lost hit@5, and an exact two-sided sign-test p-value. A failed `rerank()` is still scored on the vector-order fallback `search_codebase` returns, but counted and the `reranked` stage flagged as degraded. Result files record the search conditions (similarity threshold, `reranker.overfetch`, `reranker.candidateFloor`, and the pool sizes and HNSW settings the searches reported). `--baseline` compares per stage with the same paired counts, compares pool recall and short pools, names stages present on one side only, and warns on any differing condition or rerank failures. Result files written before this change load as a baseline in the stage their recorded reranker implies, with "no pool data" and a missing-conditions warning. Each category row shows the share of one case, so a single-case move is visible as such. No change to `search_codebase` results.
|
|
13
|
+
- **`rag-index eval` runs on frozen corpora** _(internal development tool: the eval suites, corpora and results live in this repository and are not shipped in the npm package)_ — the default suite directory `benchmarks/eval/` now holds suites written against two pinned repositories, `twenty` (Nx: NestJS + GraphQL server, React front, small packages) and `arbook-backend` (Express + Apollo, schema-first GraphQL), declared in each suite as `corpus: { root, commit }`. Before any search, eval checks that each corpus is checked out at that commit with no change to tracked files, no untracked file the indexer would collect, and untracked `.ragignore` files equal to the copies in `benchmarks/corpus/`; otherwise it exits 1 without searching or writing a result file, naming every difference. Result files record per corpus the commit, its indexed projects, chunk count and a fingerprint of the indexed chunks (positions, text, owners, embeddings), plus the number of live cases; `--baseline` warns when a corpus is on one side only, its commit or fingerprint differs, or either run includes live cases. Metrics, pool recall and stage change are also reported per corpus. The former suites on `RAG` and `fix-flow-crm-backend` moved to `benchmarks/eval-live/` (`--suite benchmarks/eval-live`) as smoke tests. Setup: keep the corpus checkouts at their commits and `rag-index --project <p> --reindex` every corpus project. While it runs, eval prints one progress line per case to stderr (`[12/153] T_ID_012 vector 3 → reranked 1 (4.2s)`).
|
|
14
|
+
- **Voyage reranker** — `reranker.provider: voyage` reranks through Voyage AI (`rerank-3` by default, `rerank-3-lite` as the cheaper alternative) with any embedder, including a local Infinity one. Its defaults are its own: `reranker.baseURL` `https://api.voyageai.com/v1`, endpoint `/v1/rerank`. The key comes only from `reranker.apiKey` — never from `embedding.apiKey` — and a Voyage reranker without one stays disabled, with a warning in the log at `logging.level` `warn` or lower; startup is unaffected.
|
|
15
|
+
- **`rag-setup` asks for a reranker for every embedding provider** — `voyage`, `openai-compatible` (base URL, model, optional key stored as `RERANKER_API_KEY`) or `none`, plus `infinity` (same container) for an Infinity embedder; the default follows the embedding provider (`infinity` → `infinity`, `voyage` → `voyage`, `openai-compatible` → `none`). The Voyage reranker key is always asked separately and stored as `VOYAGE_RERANK_API_KEY`, so it never overwrites the embedding key; a blank key offers to disable the reranker. The wizard always writes an explicit `reranker` block.
|
|
16
|
+
- **Current Voyage embedding models** — `rag-setup` suggests `voyage-code-4`, and the model registry and dashboard suggestions list `voyage-code-4`, `voyage-4-large`, `voyage-4` and `voyage-4-lite` (1024 dimensions by default, up to 2048). `voyage-code-3`, `voyage-3` and `voyage-3-lite` are no longer listed but still work — their dimension is probed at index time. The dashboard suggests `rerank-3` / `rerank-3-lite` for `reranker.model` when the saved `reranker.provider` is `voyage`.
|
|
17
|
+
|
|
18
|
+
### Changed
|
|
19
|
+
|
|
20
|
+
- **BREAKING (config): `logging.searchMetrics` is removed** — the `search metrics` line is now a plain `info` line: set `logging.level: info` to get it, and it is never written at `warn`/`error`. A config file that still contains the key loads and ignores it; `rag-config set logging.searchMetrics` fails as an unknown key.
|
|
21
|
+
- **Errors are logged as structured records under `err`** — every caught error in the log now carries `err` with its type, message, stack (up to 30 frames), driver fields (`code`, `errno`, `syscall`, `detail`, `hint`, …) and its cause chain as `causes[]` (up to 5 links); a failed database query keeps its SQL (≤ 500 characters) and parameter count with up to 30 values. `reason` now holds only domain codes, no longer the error message. The indexer's console line names the root cause: `[indexer] Failed: <file> — <message> (<code>)`.
|
|
22
|
+
- **The candidate query always fetches the `poolSaturated` probe row** — `fetchLimit + 1` rows on every search, so `hnsw.ef_search` is now `fetchLimit + 1` for pools of 40 or more (pools of 39 or fewer keep the floor of 40), whatever the logging config. Results near the threshold boundary can shift marginally; `rag-index eval --baseline` against a result taken without `searchMetrics` warns that `efSearch` differs — re-run the baselines once.
|
|
23
|
+
- **Chunks follow declarations and members** — **requires `rag-index --reindex`.** A declaration's chunk now starts at its doc comment (JSDoc or line comments directly above it, no blank line between) or its first decorator, so doc comments and decorators are embedded with the code they belong to, and — when they fit `indexing.chunkSize` together with the declaration's name line — no chunk is a lone `@Injectable()` or a doc comment cut off from the declaration below it; otherwise the decorators alone stay with the name line when they fit with it, and a longer doc comment or decorator is cut by the character splitter. Top-level declarations of at least 200 characters that share no line with another declaration get chunks of their own (enums and `declare module` blocks are packed like other module-scope code); smaller ones (type aliases, one-line constants) are packed together. A class or object literal longer than `indexing.chunkSize` is split by its members: each member of at least 200 characters that shares no line with a sibling or with the class's first or last line gets its own chunk, starting at its doc comment; the header, fields, getters and short constructors are packed together; only a member still longer than `chunkSize` is cut by the character splitter, inside itself. A class that fits `chunkSize` stays one chunk. Import and re-export statements — multi-line ones, `import x = require(…)` and `export … from` included — are removed before packing, so they are never mixed into a code chunk. Short leftover pieces without declaration code (`export default meta;`, a closing brace) are merged into a neighbouring chunk when the result fits `indexing.chunkSize`, no removed import block lies between and — when the leftover lies inside a declaration with chunks of its own, or a unit cut by the character splitter — the neighbour lies inside the same one. A short file prologue before the imports (`'use client';`, `/* @license … */`, a shebang, lint directives — under 60 characters) is removed with them; a longer file-header comment followed by imports stays a chunk of its own.
|
|
24
|
+
- **`Symbol:` in `search_codebase` names the smallest declaration containing all of the chunk's declaration code** — not the declaration of the chunk's first line, so a chunk is never labelled with a method it only overlaps. A chunk of several small members shows their class; a chunk of several small top-level declarations, or of module-scope code only, shows none. Module-scope code (top-level statements, enums, `declare module` blocks) belongs to no declaration, so a chunk packing it with one small declaration shows that declaration. The same rule applies to schema files and the `character` strategy.
|
|
25
|
+
- **`search_codebase` returns at most 2 results per file by default** — new optional `per_file` argument (integer ≥ 1, default 2) caps the results taken from one file; the remaining slots are filled from the next candidates (a larger pool without reranking, the full rerank order with it). Pass a higher `per_file` when several answers are expected in one file. `rag-index eval` is unaffected (it searches with `per_file` equal to its limit).
|
|
26
|
+
- **The MCP server's `rerank` guidance follows the re-measured eval** — `rerank: false` is still advised for "what does X do" / "where is X" lookups (reranking costs them −0.13 MRR@10), but conceptual/architectural queries are no longer described as helped by reranking: on the new chunking it is roughly neutral for them (−0.05, not significant).
|
|
27
|
+
- **`code_chunk` gains a nullable `endLine` column**, added automatically on the next `rag-index`; rows indexed before it keep `null` until re-indexed.
|
|
28
|
+
- **The reranker sees each chunk's file and owning symbol** — every document sent to the reranker starts with one line `File: <path> | Symbol: <owner>` (`File: <path>` for a chunk without an owner; line breaks in either name become spaces), so two same-named methods in different classes are no longer indistinguishable to it. Applies to every reranker provider, with no config key. Result format, stored chunks and the vector stage are unchanged; the order of reranked results can change. No re-index required.
|
|
29
|
+
- **Reranking is off by default for every non-`infinity` embedder without a `reranker` block** — when `reranker.enabled` is absent, reranking is on only with an `infinity` embedder or when the `reranker` block names a server (`provider`, `baseURL` or `url`); a block with only `model` does not turn it on. Before, any other embedding config without a `reranker` block sent every search's rerank request to the embedding host. On a cloud host (OpenAI, Voyage) that could not succeed and fell back to vector order; a self-hosted `openai-compatible` server that does serve `/rerank` (vLLM, an external Infinity) now needs `reranker.enabled: true` or a `reranker.provider`/`baseURL` to keep reranking. An explicit `reranker.enabled` behaves as before.
|
|
30
|
+
- **A malformed rerank response fails at once** — a 2xx response without the provider's result list, or with indices outside the document list, fails the rerank call with an error naming the URL instead of being retried as a network error; search falls back to vector order as for any rerank failure.
|
|
31
|
+
- **`get_dependents` / `get_dependencies` group their direct results by file** — one line per importing/imported file instead of one per `code_dependency` row, listing the distinct names imported (`default` / `* as namespace` / `side-effect` for a nameless import, re-exports grouped after plain imports under one `re-export ` prefix, `export *` for a namespace re-export, capped at 8 then `(+N more)`); `get_dependents` additionally lists up to 3 distinct enclosing declarations using the import (`` · in `PlansPage` ``, `(+N more)` beyond that) — only those resolved unambiguously, so the part can be partial or absent. The direct section header now states the file count (`**Direct:** 8 files`). Transitive sections and the empty-result/grep-fallback text are unchanged. Output format only — no MCP tool signature, config key or CLI command changes, no re-index required.
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
|
|
35
|
+
- **"MCP server is unavailable" names the cause when PostgreSQL refuses every address** — a connection refused on both `::1` and `127.0.0.1` arrives as an `AggregateError` without a message, so the tool response read `MCP server is unavailable: ` with nothing after it. It now lists the refused addresses (`connect ECONNREFUSED ::1:5432; connect ECONNREFUSED 127.0.0.1:5432`), and the log record's `err` carries each of them under `errors`.
|
|
36
|
+
- **`search_codebase` on large projects returns the configured candidate pool** — a `project`-scoped search on a project large enough for PostgreSQL to use the HNSW index (measured from ~5.8 k chunks; the crossover depends on table statistics), or any unscoped search, returned at most 40 candidates and as few as 0 even when hundreds of chunks passed the threshold: the index scan stopped at `hnsw.ef_search` (default 40) before the project filter and threshold were applied. Each search now sets `hnsw.ef_search` to the requested row count (at least 40, at most 1000) and `hnsw.iterative_scan = relaxed_order` for its own query, so the pool sized by `reranker.candidateFloor` / `reranker.overfetch` is actually fetched unless the iterative scan stops at `hnsw.max_scan_tuples` (20 000) first. Reranked results may shift, since the reranker now sees the full pool. pgvector ≥ 0.8 is recommended: an older version gets the raised `ef_search` only, with one `warn` line in the log (hidden at the default `logging.level` `error`). `search metrics` lines and `rag-index eval` search conditions record the `efSearch` / `iterativeScan` used. No config key, MCP tool signature or index change; no re-index required.
|
|
37
|
+
|
|
38
|
+
## [0.2.3-alpha] - 2026-10-21
|
|
39
|
+
|
|
40
|
+
### Measured, no change
|
|
41
|
+
|
|
42
|
+
- **Embedding prompt: members in `Symbols:` and an `Owner:` line both measured, neither shipped** — `prompt-owner-line` ran `rag-index eval` against two prompt variants (members overlapping the chunk added to `Symbols:`; a new `Owner: <ownerPath>` line) against the `retrieval-eval-set` reference baseline (41 cases). Members: overall MRR@10 +0.0192, short of the required ≥ 0.02. Owner line: overall MRR@10 -0.0112, with the `identifier` category dropping -0.0268 (over the -0.02 limit). Neither met the decision rule fixed in advance, so the embedding prompt is unchanged. Results: `benchmarks/results/prompt-owner-line-decision.md`. Separately, re-running the same suite with the reranker disabled scored far higher than with it enabled for both variants (+0.16 to +0.21 MRR@10) — unrelated to the prompt tested, but worth its own investigation into the configured reranker.
|
|
43
|
+
|
|
44
|
+
### Added
|
|
45
|
+
|
|
46
|
+
- **`rag-index --project [name]` scopes a run to one project or monorepo unit** — matched exactly against the same names `list_projects` prints; a unit name (e.g. `micros-learning`) expands to all of its children. Bypasses the `.ragprojects`/`.ragskip` selector and project pruning entirely, leaving every other indexed project untouched. Composes with `--reindex` to force a full reindex of just that project. Omitting the name opens a one-off interactive single-choice prompt (distinct from the `--select` multi-select checkbox). An unresolved name is reported with the full list of available projects and exits non-zero before any database or embedding-backend work. The post-index project graph (`project-graph.html`) now always reflects the full discovered workspace regardless of `--select`/`--project` scoping.
|
|
47
|
+
- **`search_codebase` gains a per-call `rerank` override** — pass `rerank: false` to skip reranking regardless of `reranker.enabled` (uses `fetchLimit = limit`, no candidate floor), or `rerank: true` to force it on when a reranker is configured (no-op otherwise). A 41-case / 82-query benchmark across both projects' `benchmarks/eval/*.json` suites found `rerank: false` performs same-or-better in a strong majority of identifier/behaviour/concept cases at `threshold=0.6`/`limit=5`; guidance in `McpServer.ts` and `CLAUDE.md` recommends `false` for "what does X do" / "where is X" lookups described without an exact name, and points exact-name lookups at `find_symbol` instead of `search_codebase` (the rerank setting is moot there — `find_symbol` resolved every sampled identifier exactly at rank 1).
|
|
48
|
+
- **`find_symbol` finds NestJS GraphQL operations, resolve fields and types by their GraphQL name** — `@Query`/`@Mutation`/`@Subscription`/`@ResolveField`/`@ObjectType`/`@InputType`/`@InterfaceType` decorators imported from `@nestjs/graphql` get an extra `code_symbol` row named by the GraphQL name (e.g. `adminPendingSubscriptionOrders`, not the method's TypeScript name), with new kinds `graphql_query`/`graphql_mutation`/`graphql_subscription`/`graphql_field`/`graphql_type` and a ` → \`<target>\`` in the output naming the TypeScript declaration. These rows never affect retrieval, chunk ownership or the call graph. Run `rag-index --reindex` to get the new rows for a project already indexed.
|
|
49
|
+
|
|
50
|
+
### Changed (breaking)
|
|
51
|
+
|
|
52
|
+
- **One path form for every MCP tool: workspace-relative** — the form `search_codebase`, `find_symbol` and `get_callers` already print (`my-app/src/index.ts`; for a nested-monorepo project the on-disk path, `micros-learning/apps/account/...`). `read_file` takes it instead of an absolute path; `list_files` takes a workspace directory (`.` for the root) and loses its `project` argument; `get_dependencies` / `get_dependents` lose their `project` argument (a passed `project` is ignored). An absolute path is refused with its workspace-relative equivalent, so update prompts or skills that pass absolute paths or `project` to these tools.
|
|
53
|
+
|
|
54
|
+
### Changed
|
|
55
|
+
|
|
56
|
+
- **A path that names nothing is an error, not an empty result** — every path-taking tool answers `Not found:` with the expected form (example `my-app/src` for `list_files`, `my-app/src/index.ts` elsewhere) and up to three `Did you mean:` candidates, `Not indexed:` (with the reason — excluded file type, ignored directory, `.ragignore`, unselected project, or not yet indexed), `Deleted from disk:`, `Not a file:` / `Not a directory:` or `Cannot read:`, all as MCP error responses. A database or discovery failure is an error response, never "not found".
|
|
57
|
+
- **`Did you mean:` resolves nested-monorepo project names** — a path that starts with a project's name instead of its directory (`micros-learning/account/src/main.ts`) is offered at its on-disk location (`micros-learning/apps/account/src/main.ts`) first, when that file exists; `list_files` given a project name is pointed at the project's directory.
|
|
58
|
+
- **Graph tools span projects** — `get_dependencies`, `get_dependents` and `include_graph_context` list edges into and out of other projects; an edge stored under two overlapping projects prints once; import targets outside the workspace (`../…`) are never printed.
|
|
59
|
+
- **`read_file_range` reads unindexed files** — both read tools read any file under the workspace (symlinks resolved before the boundary check; a symlinked file already in the index stays readable). `start_line` past the end and `end_line` before `start_line` are errors; an unreadable file is `Cannot read:`.
|
|
60
|
+
- **`list_projects` prints each project's directory** as a `path:` line, marked `(not on disk)` when it is gone. `list_files` prints a `**Directory:**` header and marks unreadable subdirectories `(unreadable)`.
|
|
61
|
+
- **`get_dependents` text-search fallback** runs in the project owning the file and reports `failed` / `partial` when grep cannot search every file, instead of `No results.`
|
|
62
|
+
- **Transitive `get_dependencies` / `get_dependents` traverse each reachable file once** — a breadth-first traversal replaces the recursive query, issuing one query per depth level instead of enumerating every import path. A file that is a direct dependency/dependent is no longer also listed under `**Transitive:**` at a larger depth; `via` on a tie is now the first predecessor in UTF-8 byte order, deterministic instead of arbitrary; indirect rows of equal depth are ordered by path in byte order instead of the database's default collation. Direct rows are unchanged.
|
|
63
|
+
|
|
64
|
+
### Fixed
|
|
65
|
+
|
|
66
|
+
- **Combined and side-effect imports no longer lose edges** — `import Layout, { NavItem } from './layout'` previously recorded only the named import (`NavItem`), losing the default; `import X, * as N from './m'` recorded only the default, losing the namespace import. Each part of an import declaration — default, namespace, each named import — now produces its own edge, in that order. A side-effect import (`import './x'`) now gets a new `depType` `import-side-effect`, where before it produced no edge at all, so `get_dependents` under-reported its target as unused. Default and namespace edges also get `fromSymbol` from the local name, the same rule named imports already use. The embedding prompt is unaffected — side-effect edges are excluded from it, and the other edges already deduped by file. Run `rag-index --reindex` to get the new edges for files already indexed.
|
|
67
|
+
|
|
68
|
+
### Removed
|
|
69
|
+
|
|
70
|
+
- **Per-project symbol-index version check** — `rag-index` no longer refuses to run with `Indexed before member indexing … run rag-index --reindex`, and `find_symbol`/`get_callers` no longer show that warning. Interrupting the first indexing of a project (ESC/Ctrl+C) used to trigger it on the next run even though every row was already in the current format. If you are upgrading from 0.2.0 or earlier with an existing index, run `rag-index --reindex` once yourself — it is no longer enforced. The `symbol_index_version` table is no longer created or read; an existing one is left in place, unused.
|
|
7
71
|
|
|
8
72
|
## [0.2.2-alpha] - 2026-10-02
|
|
9
73
|
|
package/README.md
CHANGED
|
@@ -7,7 +7,7 @@ Ask your AI agent things like *"where is the payment retry logic?"* or *"show me
|
|
|
7
7
|
**Why codesearch-mcp:**
|
|
8
8
|
|
|
9
9
|
- **Dramatically fewer tokens** — without RAG, answering a cross-project question means discovery calls, directory listings, and reading several wrong files before finding the right one; a workspace of 10 projects can easily mean 50 000+ lines to sift through. With RAG, one `search_codebase` call returns 5 targeted chunks (~200–500 lines total) from across *all* projects at once — the model gets the right code immediately, with no exploration overhead
|
|
10
|
-
- **Reranker narrows the result further** — a cross-encoder reranker (
|
|
10
|
+
- **Reranker narrows the result further** — a cross-encoder reranker (on by default with the local Infinity backend; local or Voyage AI) re-scores the candidate pool so the top results are the most relevant, not just the closest vector; fewer results means less context consumed per answer
|
|
11
11
|
- **Semantic, not grep** — embeddings capture intent; queries in plain English or pseudocode find the relevant code regardless of naming
|
|
12
12
|
- **Local and private** — default setup runs entirely on your machine (Infinity embeddings + PostgreSQL); no code leaves your network
|
|
13
13
|
- **Code-structure aware** — chunks align to function and class boundaries; each result shows which declaration it belongs to (`InvoiceService.create`, `collectFiles`, …), so the model understands context without reading the surrounding file
|
|
@@ -31,7 +31,7 @@ See `CHANGELOG.md` for version history.
|
|
|
31
31
|
## Requirements
|
|
32
32
|
|
|
33
33
|
- Node.js 18+
|
|
34
|
-
- PostgreSQL 15+ with pgvector (the symbol index's unique index relies on `NULLS NOT DISTINCT`, added in PostgreSQL 15)
|
|
34
|
+
- PostgreSQL 15+ with pgvector (the symbol index's unique index relies on `NULLS NOT DISTINCT`, added in PostgreSQL 15); pgvector ≥ 0.8 recommended (filtered search on large projects; older versions work with reduced recall)
|
|
35
35
|
- Docker (for local DB + Infinity) — or bring your own PostgreSQL with pgvector + an embedding backend (see [Embedding providers](#embedding-providers))
|
|
36
36
|
|
|
37
37
|
## Installation
|
|
@@ -73,8 +73,13 @@ rag-index
|
|
|
73
73
|
rag-index --select # re-run project selection
|
|
74
74
|
rag-index --reindex # force re-embed all files (use after model change)
|
|
75
75
|
rag-index --select --reindex
|
|
76
|
+
rag-index --project billing-service # scope this run to one project, by exact name
|
|
77
|
+
rag-index --reindex --project billing-service # force a full reindex of just that project
|
|
78
|
+
rag-index --project # no name — pick one from an interactive prompt
|
|
76
79
|
```
|
|
77
80
|
|
|
81
|
+
`--project` leaves every other indexed project untouched: it skips the `.ragprojects`/`.ragskip` selector entirely and never prunes projects outside the one(s) it resolves to. A name that doesn't match any discovered project or monorepo unit exits with an error before touching the database — matching exactly, no partial or fuzzy match.
|
|
82
|
+
|
|
78
83
|
## Monorepo support
|
|
79
84
|
|
|
80
85
|
If `workspaceRoot` is an [Nx](https://nx.dev) workspace (detected by `nx.json`), `rag-index` discovers each project under `apps/`, `libs/`, and `packages/` as a **separate indexed project** rather than scanning the workspace as one blob. Cross-project imports like `@scope/shared` resolve through the workspace `tsconfig.base.json` path aliases, so `get_dependencies` / `get_dependents` report edges **across** projects.
|
|
@@ -97,16 +102,18 @@ See [docs/web-dashboard.md](docs/web-dashboard.md) for the full reference.
|
|
|
97
102
|
|
|
98
103
|
## MCP Tools
|
|
99
104
|
|
|
105
|
+
Every file or directory argument is **workspace-relative** (relative to `workspaceRoot`, `/`-separated — e.g. `my-app/src/index.ts`, `.` for the root), exactly as `search_codebase` prints it, so a path copied from one tool works in every other. A path that names nothing is an error with the expected form and up to three `Did you mean:` candidates, never an empty result.
|
|
106
|
+
|
|
100
107
|
| Tool | Description |
|
|
101
108
|
|------|-------------|
|
|
102
|
-
| `search_codebase` | Semantic search across all indexed projects. Accepts `include_graph_context` to append direct imports/dependents to each result, `include_neighbors` for surrounding chunks,
|
|
103
|
-
| `list_projects` | List projects with metadata
|
|
104
|
-
| `find_symbol` | Find a declaration or member by exact name — bare, qualified (`Class.method`), quoted, or computed — faster than semantic search |
|
|
105
|
-
| `list_files` | Browse the directory tree of
|
|
106
|
-
| `read_file` | Read a complete file
|
|
107
|
-
| `read_file_range` | Read a specific line range from
|
|
108
|
-
| `get_dependencies` | Show what files a given file imports (direct or transitive) |
|
|
109
|
-
| `get_dependents` | Show which files import a given file (direct or transitive) |
|
|
109
|
+
| `search_codebase` | Semantic search across all indexed projects. Accepts `include_graph_context` to append direct imports/dependents to each result, `include_neighbors` for surrounding chunks, `threshold` to override the similarity threshold for that query, `rerank` to force reranking on/off for that call regardless of `reranker.enabled`, and `per_file` (default 2) to cap how many results come from one file |
|
|
110
|
+
| `list_projects` | List projects with metadata, chunk counts and each project's directory (`path:` line) |
|
|
111
|
+
| `find_symbol` | Find a declaration or member by exact name — bare, qualified (`Class.method`), quoted, or computed — faster than semantic search. Also finds a NestJS GraphQL operation, resolve field or type by its GraphQL name (`kind: graphql_query`/`graphql_mutation`/`graphql_subscription`/`graphql_field`/`graphql_type`) |
|
|
112
|
+
| `list_files` | Browse the directory tree of any workspace directory |
|
|
113
|
+
| `read_file` | Read a complete file under the workspace, indexed or not |
|
|
114
|
+
| `read_file_range` | Read a specific line range from any file under the workspace |
|
|
115
|
+
| `get_dependencies` | Show what files a given file imports (direct or transitive), across projects |
|
|
116
|
+
| `get_dependents` | Show which files import a given file (direct or transitive), across projects |
|
|
110
117
|
| `get_callers` | Find all call sites of a function or method by exact name (bare or qualified) |
|
|
111
118
|
|
|
112
119
|
## Project metadata
|
|
@@ -143,7 +150,7 @@ Pass `include_graph_context: true` to `search_codebase` to see direct imports an
|
|
|
143
150
|
|
|
144
151
|
Useful for immediately understanding a file's role in the dependency graph.
|
|
145
152
|
|
|
146
|
-
Each result also shows a **Symbol** breadcrumb — the declaration the chunk
|
|
153
|
+
Each result also shows a **Symbol** breadcrumb — the smallest declaration containing all of the chunk's declaration code (`InvoiceService.create`, `collectFiles`, …) — so you can see the enclosing class/method without opening the file. A chunk packing several small members shows their class. Module-scope code (top-level statements, enums, `declare module`) belongs to no declaration, so a chunk packing it with one small declaration shows that declaration; the breadcrumb is omitted for a chunk of module-scope code only, for chunks packing several small top-level declarations, and for projects indexed before this feature (re-index to populate).
|
|
147
154
|
|
|
148
155
|
## Excluding files
|
|
149
156
|
|
|
@@ -167,6 +174,12 @@ npx pino-pretty < ~/.config/codesearch-mcp/logs/indexer/current.log
|
|
|
167
174
|
# Detailed indexing diagnostics (timing per file: deps / embed / db)
|
|
168
175
|
rag-config set logging.level debug
|
|
169
176
|
|
|
177
|
+
# See what an agent called (needs logging.level info or debug), grouped by session and connection
|
|
178
|
+
jq -c 'select(.msg=="tool call") | {session, connectionId, callId, tool, args, isError, ms}' \
|
|
179
|
+
~/.config/codesearch-mcp/logs/mcp/current.log
|
|
180
|
+
jq -s 'map(select(.msg=="tool call")) | group_by(.session, .connectionId) | map({session: .[0].session, connectionId: .[0].connectionId, calls: map(.tool)})' \
|
|
181
|
+
~/.config/codesearch-mcp/logs/mcp/current.log
|
|
182
|
+
|
|
170
183
|
# Disable file logging
|
|
171
184
|
rag-config set logging.dir false
|
|
172
185
|
```
|
|
@@ -198,10 +211,10 @@ rag-config edit # open config in $EDITOR; validates on save
|
|
|
198
211
|
| `embedding.dimensions` | (derived) | Vector width override; required for models whose native width exceeds 2000 (Matryoshka). Otherwise derived from model → registry → runtime probe |
|
|
199
212
|
| `embedding.batchSize` | `32` | Texts per batch embedding request |
|
|
200
213
|
| `search.similarityThreshold` | `0.6` | Max cosine distance — raise if search returns nothing |
|
|
201
|
-
| `reranker.enabled` |
|
|
202
|
-
| `reranker.provider` | `openai-compatible` | Rerank backend — `openai-compatible` `/rerank`
|
|
203
|
-
| `reranker.baseURL` |
|
|
204
|
-
| `reranker.model` | `mixedbread-ai/mxbai-rerank-xsmall-v1`
|
|
214
|
+
| `reranker.enabled` | context | Enable cross-encoder reranking. When absent: on with `infinity` embedding or when the `reranker` block names a server (`provider`/`baseURL`/`url`), off otherwise |
|
|
215
|
+
| `reranker.provider` | `openai-compatible` | Rerank backend — `openai-compatible` `/rerank` (Infinity, vLLM) or `voyage` (Voyage AI, needs `reranker.apiKey`) |
|
|
216
|
+
| `reranker.baseURL` | per-provider | Rerank server base URL — `embedding.baseURL` for `openai-compatible` (Infinity serves embed + rerank from one server), `https://api.voyageai.com/v1` for `voyage` |
|
|
217
|
+
| `reranker.model` | per-provider | Reranker model name — `mixedbread-ai/mxbai-rerank-xsmall-v1` for `openai-compatible`, `rerank-3` for `voyage` |
|
|
205
218
|
| `reranker.overfetch` | `5` | Candidate multiplier before reranking (`limit × N`) |
|
|
206
219
|
| `reranker.candidateFloor` | `80` | Minimum candidate pool fed to the reranker regardless of `limit` — `fetchLimit = min(max(limit × overfetch, candidateFloor), 200)`. Raise on GPU/fast hosts for better recall; lower on weak CPU to cut latency |
|
|
207
220
|
| `indexing.chunkSize` | `1500` | Max characters per chunk |
|
|
@@ -211,9 +224,8 @@ rag-config edit # open config in $EDITOR; validates on save
|
|
|
211
224
|
| `indexing.chunkStrategy` | `ast` | How TS/JS files are chunked: `ast` = boundaries aligned to functions/classes, `character` = fixed-size splitter. Re-index to apply |
|
|
212
225
|
| `search.defaultLimit` | `5` | Default result count returned by `search_codebase` |
|
|
213
226
|
| `search.maxLimit` | `20` | Hard cap on result count |
|
|
214
|
-
| `logging.level` | `error` | Log level written to file: `error` \| `warn` \| `info` \| `debug` |
|
|
227
|
+
| `logging.level` | `error` | Log level written to file: `error` \| `warn` \| `info` \| `debug`. `info` adds one `tool call` line per MCP tool call and one `search metrics` line per search |
|
|
215
228
|
| `logging.dir` | `~/.config/codesearch-mcp/logs` | Base log directory, or `"false"` to disable |
|
|
216
|
-
| `logging.searchMetrics` | `false` | Emit one structured search-tuning line per query (at `info`, independent of `logging.level`) — use it to size `candidateFloor`/`overfetch`/`threshold` for your hardware |
|
|
217
229
|
|
|
218
230
|
## Embedding providers
|
|
219
231
|
|
|
@@ -227,7 +239,7 @@ rag-config edit # open config in $EDITOR; validates on save
|
|
|
227
239
|
| `openai-compatible` | OpenAI cloud, or any OpenAI-style server (TEI, vLLM, LM Studio, LocalAI, **or Ollama via its `/v1` endpoint**). | `baseURL` (+ `apiKey` for cloud) |
|
|
228
240
|
| `voyage` | Voyage AI cloud, code-specialized embeddings. | Voyage `apiKey` |
|
|
229
241
|
|
|
230
|
-
Easiest path: `rag-setup` asks for the provider, model,
|
|
242
|
+
Easiest path: `rag-setup` asks for the provider, model, any API key, and a reranker. Haven't run setup? It's the place to start. To change things later, edit config directly (below).
|
|
231
243
|
|
|
232
244
|
### Switching provider or changing the model
|
|
233
245
|
|
|
@@ -239,14 +251,14 @@ rag-config set embedding.provider openai-compatible
|
|
|
239
251
|
rag-config set embedding.baseURL https://api.openai.com/v1
|
|
240
252
|
rag-config set embedding.model text-embedding-3-small
|
|
241
253
|
rag-config set embedding.apiKey env:OPENAI_API_KEY # key lives in .env, not config.json
|
|
242
|
-
rag-config set reranker.enabled false #
|
|
254
|
+
rag-config set reranker.enabled false # only if rag-setup chose the `infinity` reranker: it stays aimed at the old server
|
|
243
255
|
rag-index --reindex
|
|
244
256
|
|
|
245
257
|
# Switch to Voyage AI
|
|
246
258
|
rag-config set embedding.provider voyage
|
|
247
|
-
rag-config set embedding.model voyage-code-
|
|
259
|
+
rag-config set embedding.model voyage-code-4
|
|
248
260
|
rag-config set embedding.apiKey env:VOYAGE_API_KEY # key lives in .env, not config.json
|
|
249
|
-
rag-config set reranker.enabled false #
|
|
261
|
+
rag-config set reranker.enabled false # only if rag-setup chose the `infinity` reranker (or configure the Voyage reranker, below)
|
|
250
262
|
rag-index --reindex
|
|
251
263
|
|
|
252
264
|
# Change only the model on the current provider
|
|
@@ -265,9 +277,9 @@ rag-index --reindex
|
|
|
265
277
|
Per-provider notes:
|
|
266
278
|
|
|
267
279
|
- **infinity** — the model set is bound at container startup, so changing `embedding.model` (or the reranker model) **recreates** the managed container on the next `rag-setup`/`rag-index`/dashboard save. Defaults: `jinaai/jina-embeddings-v2-base-code` + reranker `mixedbread-ai/mxbai-rerank-xsmall-v1`. Both models are included in the compose template and reranking is **enabled by default** (`reranker.enabled: true`).
|
|
268
|
-
- **openai-compatible** — point `embedding.baseURL` at the server; set `embedding.apiKey` (as `env:VAR`) for cloud, leave empty for local servers (TEI/vLLM/LM Studio/**Ollama**). E.g. `text-embedding-3-small`, `text-embedding-3-large`. Cloud endpoints (OpenAI, Azure) don't expose `/rerank
|
|
280
|
+
- **openai-compatible** — point `embedding.baseURL` at the server; set `embedding.apiKey` (as `env:VAR`) for cloud, leave empty for local servers (TEI/vLLM/LM Studio/**Ollama**). E.g. `text-embedding-3-small`, `text-embedding-3-large`. Cloud endpoints (OpenAI, Azure) don't expose `/rerank`, so without a `reranker` block reranking stays off unless you configure a separate rerank server; a block from `rag-setup`'s `infinity` reranker choice keeps reranking on at the old server until you set `reranker.enabled false` (see [Reranker with a cloud embedder](#reranker-with-a-cloud-or-external-embedder)).
|
|
269
281
|
- **Ollama** (via `openai-compatible`) — set `embedding.baseURL` to `http://localhost:11434/v1` and `embedding.model` to any model you've pulled (e.g. `bge-m3`). No dedicated code path, no auto-pull — pull the model yourself with `ollama pull <model>` first.
|
|
270
|
-
- **voyage** — set `embedding.apiKey` (as `env:VAR`); e.g. `voyage-code-
|
|
282
|
+
- **voyage** — set `embedding.apiKey` (as `env:VAR`); e.g. `voyage-code-4` (default in `rag-setup`), `voyage-4-large`, `voyage-4`, `voyage-4-lite`. Older names such as `voyage-code-3` still work — their dimension is probed. Without a `reranker` block, reranking stays off unless you configure one (a block from `rag-setup`'s `infinity` reranker choice keeps it on at the old server until you set `reranker.enabled false`) — e.g. the Voyage reranker (see [Reranker with a cloud embedder](#reranker-with-a-cloud-or-external-embedder)).
|
|
271
283
|
|
|
272
284
|
The vector **dimension** is derived automatically (model registry → one-shot probe); set `embedding.dimensions` only to force a Matryoshka-reduced width for models whose native size exceeds 2000.
|
|
273
285
|
|
|
@@ -284,23 +296,36 @@ The same "already running?" probe applies everywhere: a reachable backend defaul
|
|
|
284
296
|
|
|
285
297
|
### Reranker with a cloud or external embedder
|
|
286
298
|
|
|
287
|
-
|
|
299
|
+
With a cloud embedder (OpenAI, Voyage, Azure) and no `reranker` block, reranking is **off** — those endpoints don't serve `/rerank`. `rag-setup` asks for a reranker after the embedding step for every provider (`voyage`, `openai-compatible`, `none`, plus `infinity` for an Infinity embedder) and always writes an explicit `reranker` block. To configure one by hand:
|
|
288
300
|
|
|
289
|
-
|
|
301
|
+
These recipes edit whatever `reranker` block is already stored, so each one sets `reranker.enabled true` (an explicit `false` from `rag-setup`'s `none` choice would otherwise win) and clears the keys that would point the requests at the old server.
|
|
302
|
+
|
|
303
|
+
**Voyage reranker** (works with any embedder; its key is separate from the embedding key):
|
|
290
304
|
|
|
291
305
|
```bash
|
|
292
|
-
|
|
306
|
+
echo 'VOYAGE_RERANK_API_KEY=pa-…' >> ~/.config/codesearch-mcp/.env
|
|
307
|
+
rag-config set reranker.enabled true
|
|
308
|
+
rag-config set reranker.provider voyage
|
|
309
|
+
rag-config set reranker.model rerank-3 # or rerank-3-lite
|
|
310
|
+
rag-config set reranker.apiKey env:VOYAGE_RERANK_API_KEY
|
|
311
|
+
rag-config reset reranker.baseURL # a stored Infinity baseURL would win over the Voyage default
|
|
312
|
+
rag-config reset reranker.url
|
|
293
313
|
```
|
|
294
314
|
|
|
295
|
-
|
|
315
|
+
Without `reranker.apiKey` a Voyage reranker stays disabled and a warning is written to the log — visible only at `logging.level` `warn` or lower (the default `error` hides it).
|
|
316
|
+
|
|
317
|
+
**A separate `/rerank` server** (e.g. a local Infinity or vLLM instance while using Voyage/OpenAI for embeddings):
|
|
296
318
|
|
|
297
319
|
```bash
|
|
298
|
-
rag-config set
|
|
299
|
-
rag-config set
|
|
300
|
-
|
|
320
|
+
rag-config set reranker.enabled true
|
|
321
|
+
rag-config set reranker.provider openai-compatible # a stored voyage provider would send the Voyage wire format
|
|
322
|
+
rag-config set reranker.baseURL http://localhost:7997
|
|
323
|
+
rag-config set reranker.model mixedbread-ai/mxbai-rerank-xsmall-v1
|
|
324
|
+
rag-config reset reranker.url
|
|
325
|
+
rag-config reset reranker.apiKey # unless the server needs one
|
|
301
326
|
```
|
|
302
327
|
|
|
303
|
-
The embedding and reranking servers are fully independent — any combination works: Voyage embeddings + local Infinity reranker, OpenAI embeddings + remote
|
|
328
|
+
The embedding and reranking servers are fully independent — any combination works: Voyage embeddings + local Infinity reranker, OpenAI embeddings + remote vLLM reranker, etc.
|
|
304
329
|
|
|
305
330
|
### Secrets & external embedders
|
|
306
331
|
|
|
@@ -316,13 +341,15 @@ When reranking is enabled, `search_codebase` fetches a **candidate pool** by cos
|
|
|
316
341
|
- **Threshold ↔ window** — raising `search.similarityThreshold` lets *more* chunks under the bar, so the pool must be wide enough to reach them; a strict threshold with a small pool silently truncates recall. Widen `candidateFloor`/`overfetch` when you raise the threshold.
|
|
317
342
|
- **Latency** — the reranker cost grows with pool size (on CPU, tens of seconds; a rough guide is a large fixed cost plus a per-candidate increment). Bigger pool = better recall, slower query. GPU/fast hosts can raise `candidateFloor`; ultra-constrained hosts lower it.
|
|
318
343
|
|
|
344
|
+
The pool is reachable on large projects too. The HNSW index scan stops after `hnsw.ef_search` rows (PostgreSQL default 40) and only then applies the `project` filter and the threshold, so a scoped search on a project of a few thousand chunks or more (measured from ~5.8 k; the crossover depends on table statistics) — or an unscoped one — used to get at most 40 candidates, often far fewer. Each search now sets `hnsw.ef_search` to the requested row count (at least 40, at most 1000) and, on pgvector ≥ 0.8, `hnsw.iterative_scan = relaxed_order` for its own query, so `candidateFloor` / `overfetch` size the pool the reranker actually receives, unless the iterative scan stops at `hnsw.max_scan_tuples` (20 000) first. On an older pgvector only `ef_search` is raised, and one `warn` line per process names the version — or `unknown` if the first version read failed, with no later line naming it. Like the Voyage warning above, it is visible only at `logging.level` `warn` or lower.
|
|
345
|
+
|
|
319
346
|
To size these for **your** hardware, turn on telemetry:
|
|
320
347
|
|
|
321
348
|
```bash
|
|
322
|
-
rag-config set logging.
|
|
349
|
+
rag-config set logging.level info
|
|
323
350
|
```
|
|
324
351
|
|
|
325
|
-
Each search then writes one `
|
|
352
|
+
Each search then writes one `search metrics` line with, among others: `fetchLimit`, `candidatesReturned`, `poolSaturated`, `maxCosineRankSurvived`, `distinctFilesInResult`, `topFileShare`, `efSearch`/`iterativeScan` (the HNSW settings sent with the query), and `embedMs`/`rerankMs`/`dbMs`/`totalMs`. Read them as:
|
|
326
353
|
|
|
327
354
|
- **`poolSaturated: true`** — more chunks were under the threshold than the window took → recall is being truncated; raise `candidateFloor`/`overfetch`.
|
|
328
355
|
- **`maxCosineRankSurvived`** — the deepest cosine rank that survived reranking. If it stays well below `fetchLimit`, the pool is bigger than needed → shrink it to save latency; if it approaches `fetchLimit`, widen.
|
package/dist/app.js
CHANGED
|
@@ -41,24 +41,18 @@ const container_1 = require("./src/container");
|
|
|
41
41
|
const config_1 = require("./src/config");
|
|
42
42
|
const config_store_1 = require("./src/config-store");
|
|
43
43
|
const IndexCommand_1 = require("./src/commands/IndexCommand");
|
|
44
|
+
const EvalCommand_1 = require("./src/commands/EvalCommand");
|
|
44
45
|
const McpCommand_1 = require("./src/commands/McpCommand");
|
|
45
46
|
const ConfigCommand_1 = require("./src/commands/ConfigCommand");
|
|
46
47
|
const WebCommand_1 = require("./src/commands/WebCommand");
|
|
47
48
|
const setup_1 = require("./src/setup");
|
|
48
|
-
/**
|
|
49
|
-
* Load `~/.config/codesearch-mcp/.env` into `process.env` before config is read.
|
|
50
|
-
* All five binaries share this file — `rag-mcp` in particular is spawned by the
|
|
51
|
-
* MCP client and does not inherit shell exports. A real `process.env` value
|
|
52
|
-
* always wins over the file. Minimal `KEY=VALUE` parser (no `dotenv` dep):
|
|
53
|
-
* skips blanks and `#` comments, strips one layer of surrounding quotes.
|
|
54
|
-
*/
|
|
55
49
|
function loadDotEnv(file) {
|
|
56
50
|
let text;
|
|
57
51
|
try {
|
|
58
52
|
text = fs.readFileSync(file, 'utf-8');
|
|
59
53
|
}
|
|
60
54
|
catch {
|
|
61
|
-
return;
|
|
55
|
+
return;
|
|
62
56
|
}
|
|
63
57
|
for (const rawLine of text.split(/\r?\n/)) {
|
|
64
58
|
const line = rawLine.trim();
|
|
@@ -75,7 +69,7 @@ function loadDotEnv(file) {
|
|
|
75
69
|
val = val.slice(1, -1);
|
|
76
70
|
}
|
|
77
71
|
if (!(key in process.env))
|
|
78
|
-
process.env[key] = val;
|
|
72
|
+
process.env[key] = val;
|
|
79
73
|
}
|
|
80
74
|
}
|
|
81
75
|
async function main() {
|
|
@@ -86,17 +80,50 @@ async function main() {
|
|
|
86
80
|
const isSetup = argv1 === 'rag-setup' || process.argv[2] === 'setup' || process.argv[2] === 'rag-setup';
|
|
87
81
|
const isConfig = argv1 === 'rag-config' || process.argv[2] === 'config' || process.argv[2] === 'rag-config';
|
|
88
82
|
const isWeb = argv1 === 'rag-web' || process.argv[2] === 'web' || process.argv[2] === 'rag-web';
|
|
89
|
-
const mode = isIndex ? 'index' : isMcp ? 'mcp' : isSetup ? 'setup' : isConfig ? 'config' : isWeb ? 'web' : (process.argv[2] ?? '');
|
|
90
83
|
const isBinary = ['rag-index', 'rag-mcp', 'rag-setup', 'rag-config', 'rag-web'].includes(argv1);
|
|
84
|
+
const evalSubArg = isBinary ? process.argv[2] : process.argv[3];
|
|
85
|
+
const isEval = isIndex && evalSubArg === 'eval';
|
|
86
|
+
const mode = isEval ? 'eval' : isIndex ? 'index' : isMcp ? 'mcp' : isSetup ? 'setup' : isConfig ? 'config' : isWeb ? 'web' : (process.argv[2] ?? '');
|
|
87
|
+
const helpKey = mode === 'eval' ? 'eval' : (isBinary ? argv1 : mode);
|
|
91
88
|
if (process.argv.includes('--help') || process.argv.includes('-h')) {
|
|
92
|
-
(0, help_1.printHelp)(
|
|
89
|
+
(0, help_1.printHelp)(helpKey);
|
|
93
90
|
process.exit(0);
|
|
94
91
|
}
|
|
95
|
-
const effectiveArgs = process.argv.slice(isBinary ? 2 : 3);
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const
|
|
99
|
-
|
|
92
|
+
const effectiveArgs = process.argv.slice(isEval ? (isBinary ? 3 : 4) : (isBinary ? 2 : 3));
|
|
93
|
+
if (mode === 'eval') {
|
|
94
|
+
const valueFlags = new Set(['--suite', '--baseline', '--project']);
|
|
95
|
+
const unknown = [];
|
|
96
|
+
for (let i = 0; i < effectiveArgs.length; i++) {
|
|
97
|
+
const a = effectiveArgs[i];
|
|
98
|
+
if (valueFlags.has(a)) {
|
|
99
|
+
const next = effectiveArgs[i + 1];
|
|
100
|
+
if (next !== undefined && !next.startsWith('-'))
|
|
101
|
+
i++;
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
unknown.push(a);
|
|
105
|
+
}
|
|
106
|
+
if (unknown.length > 0) {
|
|
107
|
+
console.error(`Unexpected argument: ${unknown[0]}\n`);
|
|
108
|
+
(0, help_1.printHelp)(helpKey);
|
|
109
|
+
process.exit(1);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
else if (mode === 'index') {
|
|
113
|
+
const boolFlags = new Set(['--reindex', '--select']);
|
|
114
|
+
const unknown = [];
|
|
115
|
+
for (let i = 0; i < effectiveArgs.length; i++) {
|
|
116
|
+
const a = effectiveArgs[i];
|
|
117
|
+
if (boolFlags.has(a))
|
|
118
|
+
continue;
|
|
119
|
+
if (a === '--project') {
|
|
120
|
+
const next = effectiveArgs[i + 1];
|
|
121
|
+
if (next !== undefined && !next.startsWith('-'))
|
|
122
|
+
i++;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
unknown.push(a);
|
|
126
|
+
}
|
|
100
127
|
if (unknown.length > 0) {
|
|
101
128
|
console.error(`Unexpected argument: ${unknown[0]}\n`);
|
|
102
129
|
(0, help_1.printHelp)(isBinary ? argv1 : 'rag-index');
|
|
@@ -121,7 +148,7 @@ async function main() {
|
|
|
121
148
|
process.exit(1);
|
|
122
149
|
}
|
|
123
150
|
i++;
|
|
124
|
-
continue;
|
|
151
|
+
continue;
|
|
125
152
|
}
|
|
126
153
|
console.error(`Unexpected argument: ${effectiveArgs[i]}\n`);
|
|
127
154
|
(0, help_1.printHelp)(isBinary ? argv1 : 'rag-web');
|
|
@@ -136,8 +163,7 @@ async function main() {
|
|
|
136
163
|
await new ConfigCommand_1.ConfigCommand(effectiveArgs).execute();
|
|
137
164
|
process.exit(0);
|
|
138
165
|
}
|
|
139
|
-
|
|
140
|
-
const loggerName = mode === 'index' ? 'indexer' : mode === 'mcp' ? 'mcp' : mode === 'web' ? 'web' : undefined;
|
|
166
|
+
const loggerName = mode === 'index' || mode === 'eval' ? 'indexer' : mode === 'mcp' ? 'mcp' : mode === 'web' ? 'web' : undefined;
|
|
141
167
|
let container;
|
|
142
168
|
try {
|
|
143
169
|
container = (0, container_1.createContainer)(loggerName);
|
|
@@ -156,11 +182,9 @@ async function main() {
|
|
|
156
182
|
console.error(`workspaceRoot is not configured. ${hint}`);
|
|
157
183
|
process.exit(1);
|
|
158
184
|
}
|
|
159
|
-
// Construct only the selected command — building every command eagerly would
|
|
160
|
-
// run each constructor's arg parsing (e.g. WebCommand's --port) for modes that
|
|
161
|
-
// never use it.
|
|
162
185
|
const factories = {
|
|
163
186
|
index: () => new IndexCommand_1.IndexCommand(container, argv1),
|
|
187
|
+
eval: () => new EvalCommand_1.EvalCommand(container, argv1),
|
|
164
188
|
mcp: () => new McpCommand_1.McpCommand(container),
|
|
165
189
|
web: () => new WebCommand_1.WebCommand(container, effectiveArgs),
|
|
166
190
|
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Config } from '../config';
|
|
2
|
+
import type { IChunker, ChunkPiece, ChunkOpts } from '../interfaces/IChunker';
|
|
3
|
+
import type { IChunkPostProcessor, IChunkStrategy } from './interfaces';
|
|
4
|
+
export declare class AstChunker implements IChunker {
|
|
5
|
+
private readonly declarationStrategy;
|
|
6
|
+
private readonly characterStrategy;
|
|
7
|
+
private readonly postProcessors;
|
|
8
|
+
private readonly declarationPostProcessors;
|
|
9
|
+
private readonly chunkSize;
|
|
10
|
+
constructor(declarationStrategy: IChunkStrategy, characterStrategy: IChunkStrategy, postProcessors: IChunkPostProcessor[], declarationPostProcessors: IChunkPostProcessor[], config: Config);
|
|
11
|
+
chunk(code: string, opts: ChunkOpts): Promise<ChunkPiece[]>;
|
|
12
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AstChunker = void 0;
|
|
4
|
+
const DeclarationTree_1 = require("../declarations/DeclarationTree");
|
|
5
|
+
const SourceLines_1 = require("../declarations/SourceLines");
|
|
6
|
+
class AstChunker {
|
|
7
|
+
declarationStrategy;
|
|
8
|
+
characterStrategy;
|
|
9
|
+
postProcessors;
|
|
10
|
+
declarationPostProcessors;
|
|
11
|
+
chunkSize;
|
|
12
|
+
constructor(declarationStrategy, characterStrategy, postProcessors, declarationPostProcessors, config) {
|
|
13
|
+
this.declarationStrategy = declarationStrategy;
|
|
14
|
+
this.characterStrategy = characterStrategy;
|
|
15
|
+
this.postProcessors = postProcessors;
|
|
16
|
+
this.declarationPostProcessors = declarationPostProcessors;
|
|
17
|
+
this.chunkSize = config.indexer.chunkSize;
|
|
18
|
+
}
|
|
19
|
+
async chunk(code, opts) {
|
|
20
|
+
const declarations = opts.declarations ?? DeclarationTree_1.DeclarationTree.EMPTY;
|
|
21
|
+
const ctx = { lines: new SourceLines_1.SourceLines(code, { jsx: opts.jsx }), declarations, chunkSize: this.chunkSize, removed: [], splitPoints: [], characterSplit: [] };
|
|
22
|
+
const byDeclarations = !opts.isSchemaFile && opts.strategy !== 'character' && declarations.roots.length > 0;
|
|
23
|
+
let pieces = await (byDeclarations ? this.declarationStrategy : this.characterStrategy).chunk(ctx, opts);
|
|
24
|
+
const processors = byDeclarations ? [...this.postProcessors, ...this.declarationPostProcessors] : this.postProcessors;
|
|
25
|
+
for (const processor of processors)
|
|
26
|
+
pieces = processor.process(pieces, ctx);
|
|
27
|
+
return pieces;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
exports.AstChunker = AstChunker;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ChunkOpts, ChunkPiece } from '../interfaces/IChunker';
|
|
2
|
+
import type { ChunkContext, IChunkStrategy, ITextSplitter } from './interfaces';
|
|
3
|
+
export declare class CharacterChunkStrategy implements IChunkStrategy {
|
|
4
|
+
private readonly jsSplitter;
|
|
5
|
+
private readonly schemaSplitter;
|
|
6
|
+
constructor(jsSplitter: ITextSplitter, schemaSplitter: ITextSplitter);
|
|
7
|
+
chunk(ctx: ChunkContext, opts: ChunkOpts): Promise<ChunkPiece[]>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CharacterChunkStrategy = void 0;
|
|
4
|
+
class CharacterChunkStrategy {
|
|
5
|
+
jsSplitter;
|
|
6
|
+
schemaSplitter;
|
|
7
|
+
constructor(jsSplitter, schemaSplitter) {
|
|
8
|
+
this.jsSplitter = jsSplitter;
|
|
9
|
+
this.schemaSplitter = schemaSplitter;
|
|
10
|
+
}
|
|
11
|
+
chunk(ctx, opts) {
|
|
12
|
+
return (opts.isSchemaFile ? this.schemaSplitter : this.jsSplitter).split(ctx.lines.text, 1);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
exports.CharacterChunkStrategy = CharacterChunkStrategy;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { ChunkPiece } from '../interfaces/IChunker';
|
|
2
|
+
import type { ChunkContext, IChunkStrategy, ISplitPolicy } from './interfaces';
|
|
3
|
+
import { RunPacker } from './RunPacker';
|
|
4
|
+
export declare class DeclarationChunkStrategy implements IChunkStrategy {
|
|
5
|
+
private readonly policy;
|
|
6
|
+
private readonly packer;
|
|
7
|
+
constructor(policy: ISplitPolicy, packer: RunPacker);
|
|
8
|
+
chunk(ctx: ChunkContext): Promise<ChunkPiece[]>;
|
|
9
|
+
private splitNode;
|
|
10
|
+
private coversParent;
|
|
11
|
+
private splitLevel;
|
|
12
|
+
private packRuns;
|
|
13
|
+
private static isShortPrologue;
|
|
14
|
+
private static gaps;
|
|
15
|
+
}
|