codesift-mcp 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Greg Laskowski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,241 @@
+# CodeSift -- Token-efficient code intelligence for AI agents
+
+CodeSift indexes your codebase with tree-sitter AST parsing and gives AI agents 22 search/retrieval tools via CLI or MCP server. It uses 20-33% fewer tokens than raw grep/Read workflows on typical code navigation tasks.
+
+## Quick install
+
+```bash
+npm install -g codesift-mcp
+```
+
+## Quick start
+
+```bash
+# Index a project
+codesift index /path/to/project
+
+# Search for a function
+codesift symbols local/my-project "createUser" --kind function --include-source
+
+# Semantic search (requires embedding provider)
+codesift retrieve local/my-project \
+  --queries '[{"type":"semantic","query":"how does caching work?"}]'
+```
+
+## Benchmark results
+
+Measured on a real 4,127-file TypeScript codebase (70 tasks, CodeSift CLI vs Bash grep/Read).
+
+| Category | CodeSift | Bash grep | Delta |
+|----------|----------|-----------|-------|
+| Text Search | 48,930 tok | 72,993 tok | **-33%** |
+| Symbol Search | 63,829 tok | 60,282 tok | +6% |
+| File Structure | 36,580 tok | 45,489 tok | **-20%** |
+| Code Retrieval | 57,703 tok | 60,482 tok | **-5%** |
+| Relationships | 52,312 tok | 60,810 tok | **-14%** |
+| Semantic Search | 7.8/10 quality | 6.5/10 | **+20% quality** |
+
+CodeSift wins 4 of 6 categories. Symbol search is at parity (verbose output, being optimized). Relationship tracing is being rewritten for AST-level accuracy.
+
+## CLI commands
+
+### Indexing
+
+| Command | Description |
+|---------|-------------|
+| `codesift index <path>` | Index a local folder |
+| `codesift index-repo <url>` | Clone and index a remote git repository |
+| `codesift repos` | List all indexed repositories |
+| `codesift invalidate <repo>` | Clear index cache for a repository |
+
+### Search
+
+| Command | Description |
+|---------|-------------|
+| `codesift search <repo> <query>` | Full-text search across all files |
+| `codesift symbols <repo> <query>` | Search symbols by name/signature |
+
+### Outline
+
+| Command | Description |
+|---------|-------------|
+| `codesift tree <repo>` | File tree with symbol counts |
+| `codesift outline <repo> <file>` | Symbol outline of a single file |
+| `codesift repo-outline <repo>` | High-level repository outline |
+
+### Symbol retrieval
+
+| Command | Description |
+|---------|-------------|
+| `codesift symbol <repo> <id>` | Get a single symbol by ID |
+| `codesift symbols-batch <repo> <ids...>` | Get multiple symbols by ID |
+| `codesift find <repo> <query>` | Find symbol and show source |
+| `codesift refs <repo> <name>` | Find all references to a symbol |
+
+### Graph & analysis
+
+| Command | Description |
+|---------|-------------|
+| `codesift trace <repo> <name>` | Trace call chain (callers/callees) |
+| `codesift impact <repo> --since <ref>` | Blast radius of git changes |
+| `codesift context <repo> <query>` | Assemble relevant code context |
+| `codesift knowledge-map <repo>` | Module dependency map |
+
+### Diff
+
+| Command | Description |
+|---------|-------------|
+| `codesift diff <repo> --since <ref>` | Structural diff between git refs |
+| `codesift changed <repo> --since <ref>` | List changed symbols between refs |
+
+### Batch & utility
+
+| Command | Description |
+|---------|-------------|
+| `codesift retrieve <repo> --queries <json>` | Batch multiple queries in one call |
+| `codesift stats` | Show usage statistics |
+| `codesift generate-claude-md <repo>` | Generate CLAUDE.md project summary |
+
+## When to use CodeSift vs grep
+
+| Task | Best tool | Why |
+|------|-----------|-----|
+| Find text in files | `codesift search` | 33% fewer tokens, BM25 ranking |
+| Find function by name | `codesift symbols` | Returns signature + body in 1 call |
+| File structure | `codesift tree` | 20% fewer tokens, symbol counts |
+| "How does X work?" | `codesift retrieve` (semantic) | 20% better quality on concept queries |
+| Find ALL occurrences | `grep -rn` | Exhaustive, no top_k cap |
+| Count matches | `grep -c` | Simple exact count |
+| Call chain tracing | `grep -rn "fn("` | CodeSift trace is being rewritten |
+
+## MCP server
+
+CodeSift runs as an [MCP](https://modelcontextprotocol.io) server, exposing all 22 tools to AI agents like Claude.
+
+### Claude Code (CLI)
+
+Add to `~/.claude.json`:
+
+```json
+{
+  "mcpServers": {
+    "codesift": {
+      "command": "codesift-mcp"
+    }
+  }
+}
+```
+
+Or from source:
+
+```json
+{
+  "mcpServers": {
+    "codesift": {
+      "command": "node",
+      "args": ["/path/to/codesift-mcp/dist/server.js"]
+    }
+  }
+}
+```
+
+### Claude Desktop
+
+Add to `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%APPDATA%\Claude\claude_desktop_config.json` (Windows):
+
+```json
+{
+  "mcpServers": {
+    "codesift": {
+      "command": "node",
+      "args": ["/path/to/codesift-mcp/dist/server.js"]
+    }
+  }
+}
+```
+
+## Semantic search
+
+Semantic search uses embeddings to answer concept queries like "how does authentication work?" that keyword search misses.
+
+### Setup
+
+Set **one** of these environment variables:
+
+| Variable | Provider | Model |
+|----------|----------|-------|
+| `CODESIFT_VOYAGE_API_KEY` | [Voyage AI](https://voyageai.com/) | `voyage-code-3` |
+| `CODESIFT_OPENAI_API_KEY` | [OpenAI](https://openai.com/) | `text-embedding-3-small` |
+| `CODESIFT_OLLAMA_URL` | [Ollama](https://ollama.com/) (local) | `nomic-embed-text` |
+
+### Usage
+
+```bash
+# Pure semantic search
+codesift retrieve local/my-project \
+  --queries '[{"type":"semantic","query":"error handling and retry logic","top_k":10}]'
+
+# Hybrid search (semantic + BM25 text, RRF-merged)
+codesift retrieve local/my-project \
+  --queries '[{"type":"hybrid","query":"caching strategy","top_k":10}]'
+```
+
+Semantic and hybrid queries exclude test files by default to maximize token efficiency. To include test files, set `"exclude_tests": false` in the sub-query or pass `--exclude-tests=false` on the CLI.
+
+### MCP example
+
+```json
+{
+  "mcpServers": {
+    "codesift": {
+      "command": "codesift-mcp",
+      "env": {
+        "CODESIFT_OPENAI_API_KEY": "sk-..."
+      }
+    }
+  }
+}
+```
+
+## Configuration
+
+All configuration is via environment variables.
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `CODESIFT_DATA_DIR` | Storage directory for indexes | `~/.codesift` |
+| `CODESIFT_WATCH_DEBOUNCE_MS` | File watcher debounce interval | `500` |
+| `CODESIFT_DEFAULT_TOKEN_BUDGET` | Default token budget for retrieval | `8000` |
+| `CODESIFT_DEFAULT_TOP_K` | Default max results for search | `20` |
+
+## How it works
+
+1. **Indexing** -- Tree-sitter WASM grammars parse source files into ASTs. Symbol extraction produces functions, classes, methods, types, constants, etc. with signatures, docstrings, and source code.
+
+2. **BM25F search** -- Symbols are tokenized (camelCase/snake_case splitting) and indexed with field-weighted BM25 scoring. Name matches rank 3x higher than body matches.
+
+3. **Semantic search** (optional) -- Source code is chunked and embedded via the configured provider. Queries are embedded at search time and ranked by cosine similarity. Multi-sub-query decomposition with Reciprocal Rank Fusion (RRF, k=60).
+
+4. **Hybrid search** -- Combines semantic embedding similarity with BM25 text matches via RRF, getting the best of both keyword and concept search.
+
+5. **File watcher** -- chokidar watches indexed folders for changes. Modified files are re-parsed and the index is updated incrementally.
+
+## Supported languages
+
+TypeScript, JavaScript (JSX/TSX), Python, Go, Rust, Java, Ruby, PHP, Markdown, CSS, Prisma.
+
+## Development
+
+```bash
+git clone https://github.com/greglas/codesift-mcp.git
+cd codesift-mcp
+npm install
+npm run download-wasm   # Download tree-sitter WASM grammars
+npm run build           # TypeScript compilation
+npm test                # Run tests (Vitest)
+npm run test:coverage   # Coverage report
+```
+
+## License
+
+MIT

package/dist/cli/args.d.ts

@@ -0,0 +1,13 @@
+export interface ParsedArgs {
+    positional: string[];
+    flags: Record<string, string | boolean>;
+}
+export type Flags = Record<string, string | boolean>;
+export declare function parseArgs(args: string[]): ParsedArgs;
+export declare function getFlag(flags: Flags, name: string): string | undefined;
+export declare function getBoolFlag(flags: Flags, name: string): boolean | undefined;
+export declare function getNumFlag(flags: Flags, name: string): number | undefined;
+export declare function output(data: unknown, flags: Flags): void;
+export declare function die(message: string): never;
+export declare function requireArg(args: string[], index: number, name: string): string;
+//# sourceMappingURL=args.d.ts.map

package/dist/cli/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC;CACzC;AAED,MAAM,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC;AAErD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CA4BpD;AAED,wBAAgB,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAItE;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAM3E;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAQzE;AAMD,wBAAgB,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAIxD;AAED,wBAAgB,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,KAAK,CAG1C;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAM9E"}

package/dist/cli/args.js

@@ -0,0 +1,79 @@
+// ---------------------------------------------------------------------------
+// CLI argument parsing and output helpers
+// ---------------------------------------------------------------------------
+export function parseArgs(args) {
+    const positional = [];
+    const flags = {};
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if (arg.startsWith("--")) {
+            const key = arg.slice(2);
+            // Handle --key=value syntax
+            if (key.includes("=")) {
+                const eqIdx = key.indexOf("=");
+                flags[key.slice(0, eqIdx)] = key.slice(eqIdx + 1);
+            }
+            else {
+                const next = args[i + 1];
+                // Boolean flags: no next value, or next value is also a flag
+                if (next === undefined || next.startsWith("--")) {
+                    flags[key] = true;
+                }
+                else {
+                    flags[key] = next;
+                    i++;
+                }
+            }
+        }
+        else {
+            positional.push(arg);
+        }
+    }
+    return { positional, flags };
+}
+export function getFlag(flags, name) {
+    const val = flags[name];
+    if (val === undefined || typeof val === "boolean")
+        return undefined;
+    return val;
+}
+export function getBoolFlag(flags, name) {
+    const val = flags[name];
+    if (val === undefined)
+        return undefined;
+    if (val === true || val === "true")
+        return true;
+    if (val === "false")
+        return false;
+    return true;
+}
+export function getNumFlag(flags, name) {
+    const raw = getFlag(flags, name);
+    if (raw === undefined)
+        return undefined;
+    const num = Number(raw);
+    if (isNaN(num)) {
+        die(`Invalid number for --${name}: ${raw}`);
+    }
+    return num;
+}
+// ---------------------------------------------------------------------------
+// Output helpers
+// ---------------------------------------------------------------------------
+export function output(data, flags) {
+    const compact = getBoolFlag(flags, "compact");
+    const indent = compact ? undefined : 2;
+    process.stdout.write(JSON.stringify(data, null, indent) + "\n");
+}
+export function die(message) {
+    process.stderr.write(`Error: ${message}\n`);
+    process.exit(1);
+}
+export function requireArg(args, index, name) {
+    const val = args[index];
+    if (val === undefined) {
+        die(`Missing required argument: <${name}>`);
+    }
+    return val;
+}
+//# sourceMappingURL=args.js.map

package/dist/cli/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,0CAA0C;AAC1C,8EAA8E;AAS9E,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,KAAK,GAAU,EAAE,CAAC;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACrB,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,4BAA4B;YAC5B,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC/B,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACzB,6DAA6D;gBAC7D,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChD,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;gBACpB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;oBAClB,CAAC,EAAE,CAAC;gBACN,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,KAAY,EAAE,IAAY;IAChD,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACxB,IAAI,GAAG,KAAK,SAAS,IAAI,OAAO,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACpE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAY,EAAE,IAAY;IACpD,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IACxB,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACxC,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,GAAG,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IAClC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAY,EAAE,IAAY;IACnD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACjC,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACxB,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACf,GAAG,CAAC,wBAAwB,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,UAAU,MAAM,CAAC,IAAa,EAAE,KAAY;IAChD,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,OAAe;IACjC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,OAAO,IAAI,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAc,EAAE,KAAa,EAAE,IAAY;IACpE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IACxB,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,GAAG,CAAC,+BAA+B,IAAI,GAAG,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/cli/commands.d.ts

@@ -0,0 +1,4 @@
+import type { Flags } from "./args.js";
+export type CommandHandler = (args: string[], flags: Flags) => Promise<void>;
+export declare const COMMAND_MAP: Record<string, CommandHandler>;
+//# sourceMappingURL=commands.d.ts.map

package/dist/cli/commands.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../src/cli/commands.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAGvC,MAAM,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAgX7E,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAuBtD,CAAC"}