@nithin554/code-crawler 0.0.0 → 0.1.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nithin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,45 +1,221 @@
-# @nithin554/code-crawler
-
-## ⚠️ IMPORTANT NOTICE ⚠️
-
-**This package is created solely for the purpose of setting up OIDC (OpenID Connect) trusted publishing with npm.**
-
-This is **NOT** a functional package and contains **NO** code or functionality beyond the OIDC setup configuration.
-
-## Purpose
-
-This package exists to:
-1. Configure OIDC trusted publishing for the package name `@nithin554/code-crawler`
-2. Enable secure, token-less publishing from CI/CD workflows
-3. Establish provenance for packages published under this name
-
-## What is OIDC Trusted Publishing?
-
-OIDC trusted publishing allows package maintainers to publish packages directly from their CI/CD workflows without needing to manage npm access tokens. Instead, it uses OpenID Connect to establish trust between the CI/CD provider (like GitHub Actions) and npm.
-
-## Setup Instructions
-
-To properly configure OIDC trusted publishing for this package:
-
-1. Go to [npmjs.com](https://www.npmjs.com/) and navigate to your package settings
-2. Configure the trusted publisher (e.g., GitHub Actions)
-3. Specify the repository and workflow that should be allowed to publish
-4. Use the configured workflow to publish your actual package
-
-## DO NOT USE THIS PACKAGE
-
-This package is a placeholder for OIDC configuration only. It:
-- Contains no executable code
-- Provides no functionality
-- Should not be installed as a dependency
-- Exists only for administrative purposes
-
-## More Information
-
-For more details about npm's trusted publishing feature, see:
-- [npm Trusted Publishing Documentation](https://docs.npmjs.com/generating-provenance-statements)
-- [GitHub Actions OIDC Documentation](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect)
-
----
-
-**Maintained for OIDC setup purposes only**
+# Code-Crawler
+
+**Static call tracing MCP server backed by a persistent property graph of your codebase.**
+
+Code-Crawler indexes a project into a SQLite-backed property graph — files, packages,
+classes, methods, fields, imports, and **call edges** — then exposes that graph to LLM
+clients as MCP tools: find callers, trace call paths, explore neighborhoods, and search
+symbols across the whole codebase.
+
+```
+        ┌───────────────────────  MCP (stdio)  ───────────────────────┐
+        │  crawl_project  search_symbols  get_callers  get_callees     │
+        │  trace_path     get_subgraph   get_node      get_schema ...  │
+        └───────────────┬──────────────────────────────┬───────────────┘
+                        │                              │
+                 ┌──────▼───────┐             ┌────────▼────────┐
+                 │   crawler    │             │  graph queries  │
+                 │ parse+resolve│             │ recursive CTEs  │
+                 └──────┬───────┘             └────────┬────────┘
+                        │                              │
+                 ┌──────▼──────────────────────────────▼──────┐
+                 │        SQLite property graph (WAL)          │
+                 │   nodes (kind, fqn, props) + edges (kind)   │
+                 └─────────────────────────────────────────────┘
+```
+
+## Why a graph?
+
+Plain-text grep can't answer questions like:
+
+- *"Who transitively calls `OrderRepository#create`?"*
+- *"Is there a static path from `Main#main` to `Tracking#dispatch`?"*
+- *"What is the impact radius of changing `OrderService#place`?"*
+
+These need graph traversal. Code-Crawler builds the graph once (idempotently, file-by-file)
+and answers them instantly with bounded recursive SQL queries.
+
+## Features
+
+- **Java-first** via the `tree-sitter-java` grammar (WASM — no native build step).
+  The parser/resolver layer is designed so more languages plug in per-language config modules.
+- **Call resolution heuristics** that go beyond naive string matching:
+  receiver types inferred from locals, parameters, fields, `new T()` expressions,
+  bare/`this`/`super` calls, static/qualified calls, and superclass chains.
+  Unresolvable calls become `unknown_symbol` phantom nodes with `resolved:false` edges,
+  so the graph stays traversable.
+- **Idempotent, file-scoped indexing** — re-crawling refreshes changed files and removes
+  stale ones without duplicating the graph.
+- **Self-contained**: single SQLite file, no external services.
+- **LLM-friendly**: every tool returns structured JSON, unknown symbols come back with
+  fuzzy search suggestions.
+
+## Install & build
+
+Requires Node.js ≥ 20.
+
+```bash
+npm install
+npm run build        # tsc → dist/
+npm test             # vitest (53 tests)
+```
+
+## CLI
+
+```bash
+codecrawler crawl <path>     # index a project
+codecrawler status           # index summary
+codecrawler query <symbol>   # fuzzy-lookup a symbol
+codecrawler serve            # start the MCP server over stdio (default)
+```
+
+Environment variables:
+
+| Variable | Default | Purpose |
+| --- | --- | --- |
+| `CODECRAWLER_DB` | `~/.code-crawler/index.db` | Database path |
+| `CODECRAWLER_EXCLUDE_DIRS` | `.git,node_modules,dist,build,target,...` | Dirs skipped during crawl |
+| `CODECRAWLER_DEFAULT_LIMIT` | `50` | Default result cap |
+| `CODECRAWLER_MAX_DEPTH` | `10` | Traversal depth cap |
+
+## CI & publishing
+
+GitHub Actions runs on every pull request (`pull-request.yml`): `npm ci` →
+typecheck → tests → build → smoke-test of the built CLI.
+
+Merges to `main` trigger `publish.yml`, which re-runs the same checks and then,
+if green, **auto-releases**:
+
+1. Reads the most recent `v*` tag (or `package.json` version on the first run),
+   computes the next patch version, and pushes a `vX.Y.Z` tag.
+2. Creates a matching GitHub release.
+3. Publishes the package to the npm registry with **provenance** (OIDC
+   trusted publishing) — no token or repository secret is needed.
+
+## Using as an MCP server
+
+### Claude Desktop
+
+`claude_desktop_config.json`:
+
+```json
+{
+  "mcpServers": {
+    "code-crawler": {
+      "command": "node",
+      "args": ["/absolute/path/to/code-crawler/dist/index.js", "serve"],
+      "env": { "CODECRAWLER_DB": "/absolute/path/to/your/index.db" }
+    }
+  }
+}
+```
+
+### Cursor / other stdio MCP clients
+
+Point the client at the same command. The typical flow for an LLM agent:
+
+1. `crawl_project` the repo once.
+2. `search_symbols` to locate the symbol of interest.
+3. `get_callers` / `get_callees` / `trace_path` to reason about the call graph.
+
+## MCP surface
+
+**Tools**
+
+| Tool | Purpose |
+| --- | --- |
+| `crawl_project` | Index a directory (idempotent) |
+| `index_status` | Node/edge/file counts + last crawl |
+| `search_symbols` | Fuzzy lookup by name or FQN, optional kind filter |
+| `get_node` | Metadata + source location for one symbol |
+| `get_callers` | Direct or transitive callers (depth-bounded) |
+| `get_callees` | Direct or transitive callees (depth-bounded) |
+| `trace_path` | Static call paths between two symbols (BFS, shortest first) |
+| `get_subgraph` | Neighborhood extraction for visualization |
+| `get_schema` | Describe the graph model |
+| `clear_index` | Reset the database |
+
+**Resources**: `codecrawler://schema`, `codecrawler://status`
+**Prompt**: `trace-call-path(from, to)`
+
+## Graph model
+
+**Nodes** — `kind` is one of `file | package | class | interface | enum | method |
+constructor | field | import | unknown_symbol`. Every node has a globally unique
+`qualifiedName` and a JSON `properties` blob (signature, modifiers, lines, ...).
+
+**Edges** — `kind` is one of `CALLS | DECLARED_IN | CONTAINS | IMPLEMENTS | EXTENDS |
+IMPORTS | REFERENCES`. `CALLS` edges carry `line`, `resolved`, and `confidence`.
+
+Id scheme:
+
+| Symbol | Example |
+| --- | --- |
+| type | `com.acme.OrderService` |
+| method | `com.acme.OrderService#place` |
+| constructor | `com.acme.OrderService#<init>` |
+| field | `com.acme.OrderService$repo` |
+| import | `__import__:java.util.List` |
+| unresolved call | `__unknown__:System.out.println` |
+
+## Java resolution heuristics
+
+1. **Constructor calls** (`new Foo(...)`) → resolve `Foo` → `#<init>` (default ctor if implicit).
+2. **Bare / `this.` / `super.` calls** → enclosing type + superclass chain.
+3. **Type-qualified calls** (`com.acme.Foo.bar`) → resolve the type directly.
+4. **Instance calls** (`x.method()`) → infer receiver type from locals → params → fields,
+   then search that type and its ancestors.
+5. **Otherwise** → phantom `unknown_symbol` node, `resolved:false`, with the inference
+   failure reason attached to the edge.
+
+Each resolved edge carries a `confidence` score and a human-readable `reason` trace
+(e.g. `via local orders: OrderService`), so LLM clients can weigh how much to trust it.
+
+## Roadmap: multi-language
+
+The parsing layer is language-agnostic. Adding a language requires:
+
+1. `src/parser/languages/<lang>.ts` — a `LanguageConfig` implementing `parseFile`
+   (package/module extraction, type/member extraction, raw call sites, local type tables).
+2. Register it in `src/parser/languages/index.ts`.
+3. `src/resolver/<lang>Resolver.ts` — translate raw call sites into resolved targets
+   using the shared `TypeIndex`.
+
+Natural next targets: **Python** (native AST or tree-sitter-python), **TypeScript/JavaScript**
+(tree-sitter-typescript), and **Go** (tree-sitter-go). The graph schema, store, traversal
+queries, and MCP surface all stay the same.
+
+Planned enhancements:
+
+- Package-level dependency graphs (module boundaries, cycle detection).
+- Global call-graph analytics: hotspots (most-called methods), dead-code candidates
+  (methods with no incoming CALLS edges), fan-in/fan-out metrics.
+- Test coverage mapping (which tests exercise a given symbol).
+- Incremental crawling (watch mode) and git-blame aware invalidation.
+
+## Architecture
+
+```
+src/
+├── index.ts            CLI (crawl | status | query | serve)
+├── server.ts           MCP server: tools, resources, prompt (thin handlers)
+├── crawler.ts          file discovery → parse → index → resolve → persist
+├── config.ts           env-overridable configuration
+├── graph/
+│   ├── schema.ts       node/edge kinds, id scheme, property shapes
+│   ├── store.ts        better-sqlite3 schema + idempotent file-scoped upserts
+│   └── queries.ts      recursive-CTE traversal (callers/callees/path/subgraph)
+├── parser/
+│   ├── language.ts     LanguageConfig contract
+│   ├── treeSitter.ts   WASM runtime + grammar loading + CST walk helpers
+│   └── languages/
+│       ├── index.ts    language registry
+│       └── java.ts     Java structural extraction
+└── resolver/
+    ├── types.ts        TypeIndex (global name/method index) + ResolvedCall
+    └── javaResolver.ts Java call-target resolution heuristics
+```
+
+## License
+
+MIT

package/dist/config.d.ts

@@ -0,0 +1,13 @@
+export interface Config {
+    /** Path to the SQLite property-graph database. */
+    dbPath: string;
+    /** Directory names excluded during crawling. */
+    excludeDirs: string[];
+    /** Default number of results for list-returning tools. */
+    defaultLimit: number;
+    /** Cap on transitive traversal depth to keep queries bounded. */
+    maxDepth: number;
+    /** When true, errors include symbol-search suggestions. */
+    helpfulErrors: boolean;
+}
+export declare function loadConfig(): Config;

package/dist/config.js

@@ -0,0 +1,32 @@
+import path from "node:path";
+import os from "node:os";
+/**
+ * Global configuration for code-crawler.
+ *
+ * All values can be overridden via environment variables so MCP clients can
+ * point the server at different databases without code changes.
+ */
+const DEFAULT_DB_PATH = path.join(os.homedir(), ".code-crawler", "index.db");
+function envStr(name, fallback) {
+    const v = process.env[name];
+    return v && v.length > 0 ? v : fallback;
+}
+function envBool(name, fallback) {
+    const v = process.env[name];
+    if (v === undefined)
+        return fallback;
+    return v === "1" || v.toLowerCase() === "true";
+}
+export function loadConfig() {
+    return {
+        dbPath: envStr("CODECRAWLER_DB", DEFAULT_DB_PATH),
+        excludeDirs: envStr("CODECRAWLER_EXCLUDE_DIRS", ".git,node_modules,dist,build,target,.idea,.venv,.gradle,out,.next,coverage")
+            .split(",")
+            .map((s) => s.trim())
+            .filter((s) => s.length > 0),
+        defaultLimit: Number(envStr("CODECRAWLER_DEFAULT_LIMIT", "50")),
+        maxDepth: Number(envStr("CODECRAWLER_MAX_DEPTH", "10")),
+        helpfulErrors: envBool("CODECRAWLER_HELPFUL_ERRORS", true),
+    };
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB;;;;;GAKG;AAEH,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;AAE7E,SAAS,MAAM,CAAC,IAAY,EAAE,QAAgB;IAC5C,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC5B,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC1C,CAAC;AAED,SAAS,OAAO,CAAC,IAAY,EAAE,QAAiB;IAC9C,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACrC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;AACjD,CAAC;AAeD,MAAM,UAAU,UAAU;IACxB,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,gBAAgB,EAAE,eAAe,CAAC;QACjD,WAAW,EAAE,MAAM,CACjB,0BAA0B,EAC1B,4EAA4E,CAC7E;aACE,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAC9B,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,2BAA2B,EAAE,IAAI,CAAC,CAAC;QAC/D,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;QACvD,aAAa,EAAE,OAAO,CAAC,4BAA4B,EAAE,IAAI,CAAC;KAC3D,CAAC;AACJ,CAAC"}

package/dist/crawler.d.ts

@@ -0,0 +1,46 @@
+import type { GraphStore } from "./graph/store.js";
+import type { GraphEdge, GraphNode } from "./graph/schema.js";
+import type { ParsedFile } from "./parser/language.js";
+import { buildTypeIndex } from "./resolver/types.js";
+/**
+ * Project crawler: discover source files, parse them, build a global type
+ * index, resolve call targets, and persist an idempotent file-scoped graph.
+ */
+export interface CrawlOptions {
+    /** Force a specific language regardless of extension. */
+    language?: string;
+    /** Optional include filter: path must contain this substring. */
+    fileFilter?: string;
+}
+export interface CrawlResult {
+    projectPath: string;
+    filesScanned: number;
+    filesParsed: number;
+    filesRemoved: number;
+    nodes: number;
+    edges: number;
+    resolvedCalls: number;
+    unresolvedCalls: number;
+    languages: string[];
+    durationMs: number;
+}
+/** Discover source files under a project, respecting exclusion dirs. */
+export declare function discoverSourceFiles(projectPath: string, excludeDirs: string[], language?: string): string[];
+/** Convert a ParsedFile into graph nodes + edges (declarations only). */
+export declare function buildDeclarationGraph(parsed: ParsedFile): {
+    nodes: GraphNode[];
+    edges: GraphEdge[];
+};
+/** Resolve call sites for a parsed file into CALLS edges + phantom nodes. */
+export declare function buildCallEdges(parsed: ParsedFile, index: ReturnType<typeof buildTypeIndex>, knownNodeIds: Set<string>): {
+    edges: GraphEdge[];
+    phantomNodes: GraphNode[];
+    resolved: number;
+    unresolved: number;
+};
+/**
+ * Crawl a project directory and index it into the store.
+ * Idempotent: re-crawling refreshes the graph; files that disappeared are
+ * removed from the index.
+ */
+export declare function crawlProject(store: GraphStore, projectPath: string, excludeDirs: string[], options?: CrawlOptions): Promise<CrawlResult>;