@nithin554/code-crawler 0.1.55 → 0.1.56

package/README.md

@@ -107,7 +107,7 @@ Requires Node.js ≥ 20.
 ```bash
 npm install
 npm run build        # tsc → dist/
-npm test             # vitest (174 tests; run with CODECRAWLER_BENCH=1 for the scale test)
+npm test             # vitest (215 tests; run with CODECRAWLER_BENCH=1 for the scale test)
 npm run bench        # crawl/query benchmark on a generated ~137k-line project
 ```
 
@@ -321,6 +321,11 @@ See [`USAGE.md`](USAGE.md) for the full task-oriented walkthrough.
 | `find_injections` | DI-supplied dependencies (`@Autowired`, NestJS, FastAPI `Depends`) |
 | `find_reflections` | Run-time lookups: `Class.forName`, `getattr`, `importlib`, dynamic `import()` |
 | `find_external_dependencies` | Third-party packages with reference and file counts |
+| `find_unresolved` | Call sites the resolver could not bind, grouped by failure reason |
+| `impact_analysis` | Callers + entry points + routes + overrides + risk band in one call |
+| `explain_symbol` | One-shot digest: members, decorators, callers, callees, wiring |
+| `find_cycles` | Call cycles (SCCs) with a sample closed path, plus direct recursion |
+| `export_graph` | Neighborhood rendered as Mermaid or Graphviz DOT |
 | `get_subgraph` | Neighborhood extraction for visualization |
 | `get_schema` | Describe the graph model |
 | `clear_index` | Reset the database |
@@ -328,6 +333,20 @@ See [`USAGE.md`](USAGE.md) for the full task-oriented walkthrough.
 **Resources**: `codecrawler://schema`, `codecrawler://status`
 **Prompt**: `trace-call-path(from, to)`
 
+**Token budgeting.** The 18 list-returning query tools — every tool whose payload is a
+single result list — accept `max_tokens` (100–200000) and `cursor`. Passing either adds a
+`page` block with `totalCount`, `returnedCount`, `truncated`, `estimatedTokens`, and
+`nextCursor`; omit both and the response shape is unchanged, so existing clients keep
+working. Paging never returns an empty page while items remain, so a continuation loop
+always terminates. Token counts are estimates from JSON length (~4 characters per token) —
+no tokenizer dependency, deterministic across models.
+
+`get_subgraph` is the one exception: its `nodes` and `edges` are two views of the same slice,
+so cutting one list would leave edges pointing at omitted nodes. It is bounded by `radius` and
+`limit` instead. Aggregate tools (`impact_analysis`, `explain_symbol`, `export_graph`) and
+single-object tools (`get_node`, `index_status`, `get_schema`) return one object and take no
+budget.
+
 ## Graph model
 
 **Nodes** — `kind` is one of `file | package | module | class | interface | enum | method |

package/dist/graph/budget.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Token budgeting and cursor pagination for tool responses.
+ *
+ * Token counts are *estimates*: JSON payloads average roughly four characters
+ * per token for source-code-adjacent text. Estimating avoids a tokenizer
+ * dependency and keeps responses deterministic across models. The budget only
+ * decides where to cut a list; it is never a hard failure condition.
+ */
+export interface BudgetOptions {
+    /** Soft cap on the estimated tokens the returned items may cost. */
+    maxTokens?: number;
+    /** Zero-based offset into the item list. */
+    cursor?: number;
+}
+export interface BudgetPage<T> {
+    items: T[];
+    /** Number of items available in the window handed to applyBudget. */
+    totalCount: number;
+    returnedCount: number;
+    cursor: number;
+    /** Offset to pass as `cursor` to fetch the next page. */
+    nextCursor?: number;
+    truncated: boolean;
+    estimatedTokens: number;
+}
+/** Metadata describing a page, without the items themselves. */
+export interface PageMeta {
+    totalCount: number;
+    returnedCount: number;
+    cursor: number;
+    nextCursor?: number;
+    truncated: boolean;
+    estimatedTokens: number;
+}
+export declare function estimateTokens(value: unknown): number;
+/**
+ * Slice a list to a token budget, starting at `cursor`.
+ *
+ * A single item is always returned even when it exceeds the budget, so a page
+ * can never come back empty while items remain.
+ */
+export declare function applyBudget<T>(items: readonly T[], opts?: BudgetOptions): BudgetPage<T>;
+/** Strip the items from a page so it can be attached as response metadata. */
+export declare function pageMeta<T>(page: BudgetPage<T>): PageMeta;
+/** True when the caller opted into budgeted output in any way. */
+export declare function budgetRequested(opts: BudgetOptions): boolean;

package/dist/graph/budget.js

@@ -0,0 +1,72 @@
+/**
+ * Token budgeting and cursor pagination for tool responses.
+ *
+ * Token counts are *estimates*: JSON payloads average roughly four characters
+ * per token for source-code-adjacent text. Estimating avoids a tokenizer
+ * dependency and keeps responses deterministic across models. The budget only
+ * decides where to cut a list; it is never a hard failure condition.
+ */
+export function estimateTokens(value) {
+    if (value === undefined)
+        return 0;
+    const text = JSON.stringify(value) ?? String(value);
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Slice a list to a token budget, starting at `cursor`.
+ *
+ * A single item is always returned even when it exceeds the budget, so a page
+ * can never come back empty while items remain.
+ */
+export function applyBudget(items, opts = {}) {
+    const totalCount = items.length;
+    const cursor = Math.max(0, Math.floor(opts.cursor ?? 0));
+    const available = cursor >= totalCount ? [] : items.slice(cursor);
+    const maxTokens = opts.maxTokens;
+    if (maxTokens === undefined || maxTokens <= 0) {
+        return {
+            items: available,
+            totalCount,
+            returnedCount: available.length,
+            cursor,
+            truncated: false,
+            estimatedTokens: estimateTokens(available),
+        };
+    }
+    const page = [];
+    let used = 0;
+    for (const item of available) {
+        const cost = estimateTokens(item);
+        if (page.length > 0 && used + cost > maxTokens)
+            break;
+        page.push(item);
+        used += cost;
+    }
+    const nextCursor = cursor + page.length;
+    const truncated = nextCursor < totalCount;
+    return {
+        items: page,
+        totalCount,
+        returnedCount: page.length,
+        cursor,
+        ...(truncated ? { nextCursor } : {}),
+        truncated,
+        estimatedTokens: used,
+    };
+}
+/** Strip the items from a page so it can be attached as response metadata. */
+export function pageMeta(page) {
+    return {
+        totalCount: page.totalCount,
+        returnedCount: page.returnedCount,
+        cursor: page.cursor,
+        ...(page.nextCursor !== undefined ? { nextCursor: page.nextCursor } : {}),
+        truncated: page.truncated,
+        estimatedTokens: page.estimatedTokens,
+    };
+}
+/** True when the caller opted into budgeted output in any way. */
+export function budgetRequested(opts) {
+    return opts.maxTokens !== undefined || opts.cursor !== undefined;
+}
+//# sourceMappingURL=budget.js.map

package/dist/graph/budget.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"budget.js","sourceRoot":"","sources":["../../src/graph/budget.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AA+BH,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IACpD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAI,KAAmB,EAAE,IAAI,GAAkB,EAAE;IAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,MAAM,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAEjC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QAC9C,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,UAAU;YACV,aAAa,EAAE,SAAS,CAAC,MAAM;YAC/B,MAAM;YACN,SAAS,EAAE,KAAK;YAChB,eAAe,EAAE,cAAc,CAAC,SAAS,CAAC;SAC3C,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAQ,EAAE,CAAC;IACrB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,GAAG,IAAI,GAAG,SAAS;YAAE,MAAM;QACtD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,IAAI,IAAI,IAAI,CAAC;IACf,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IACxC,MAAM,SAAS,GAAG,UAAU,GAAG,UAAU,CAAC;IAC1C,OAAO;QACL,KAAK,EAAE,IAAI;QACX,UAAU;QACV,aAAa,EAAE,IAAI,CAAC,MAAM;QAC1B,MAAM;QACN,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpC,SAAS;QACT,eAAe,EAAE,IAAI;KACtB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CAAI,IAAmB;IAC7C,OAAO;QACL,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,eAAe,EAAE,IAAI,CAAC,eAAe;KACtC,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,eAAe,CAAC,IAAmB;IACjD,OAAO,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC;AACnE,CAAC"}

package/dist/graph/insights.d.ts

@@ -0,0 +1,217 @@
+/**
+ * Higher-level analyses built on top of the primitive queries.
+ *
+ * These power the "agent-facing" tools: what the index could not resolve, what
+ * breaks if a symbol changes, a one-shot explanation of a symbol, call cycles,
+ * and graph export in diagram formats.
+ */
+import type { GraphStore } from "./store.js";
+import type { EdgeKind, NodeKind } from "./schema.js";
+import { type DecoratorInfo, type SymbolRef, type WiringEdge } from "./queries.js";
+export interface UnresolvedCall {
+    caller: SymbolRef;
+    /** Placeholder node id the call edge points at (`__unknown__:<label>`). */
+    targetId: string;
+    /** Callee as written in source, e.g. `System.out.println`. */
+    targetLabel: string;
+    /** Why the resolver gave up. */
+    reason: string;
+    line?: number;
+    edgeKind: EdgeKind;
+}
+export interface ReasonCount {
+    reason: string;
+    count: number;
+}
+export interface UnresolvedReport {
+    /** Total unresolved call edges matching the filters, ignoring `limit`. */
+    total: number;
+    /** Unresolved edges grouped by resolver reason, most frequent first. */
+    byReason: ReasonCount[];
+    calls: UnresolvedCall[];
+    /** True when `calls` hit the scan limit before `total` did. */
+    scanTruncated: boolean;
+    /** `kind` filter that was applied to the caller node kind. */
+    callerKind?: NodeKind;
+    /** `reason` filter that was applied (substring, case-insensitive). */
+    reasonFilter?: string;
+}
+export interface UnresolvedQueryOptions {
+    /** Substring match against the resolver reason, case-insensitive. */
+    reason?: string;
+    /** Restrict to unresolved calls made by callers of this node kind. */
+    kind?: NodeKind;
+    /** Cap on rows scanned (default 2000). */
+    limit?: number;
+}
+/**
+ * Every call the resolver could not bind to a node, with the reason it failed.
+ *
+ * Unresolved calls are the honest boundary of a static index: they show where
+ * analysis stops (dynamic dispatch, untyped receivers, missing dependencies)
+ * and they are the highest-signal input for improving the parsers.
+ */
+export declare function findUnresolved(store: GraphStore, opts?: UnresolvedQueryOptions): UnresolvedReport;
+export interface NodeUnresolvedInfo {
+    /** Calls this node makes that could not be resolved. */
+    outgoing: UnresolvedCall[];
+    /** Unresolved calls elsewhere in the graph that point at this node. */
+    incoming: {
+        caller: SymbolRef;
+        line?: number;
+    }[];
+}
+/**
+ * Unresolved call detail for a single node, used to annotate `get_node`.
+ * Returns `undefined` when the node has no unresolved edges in either
+ * direction so callers can omit the block entirely.
+ */
+export declare function unresolvedForNode(store: GraphStore, nodeId: string): NodeUnresolvedInfo | undefined;
+export interface ImpactCaller {
+    caller: SymbolRef;
+    depth: number;
+    callSiteLine?: number;
+    edgeKind: EdgeKind;
+}
+export interface ImpactImplementation {
+    type: SymbolRef;
+    /** Override of the analyzed method, when the symbol is a member. */
+    method?: SymbolRef;
+    depth: number;
+    via: "EXTENDS" | "IMPLEMENTS";
+}
+export interface ImpactSummary {
+    totalCallers: number;
+    directCallers: number;
+    entryPoints: number;
+    routes: number;
+    implementations: number;
+    affectedFiles: number;
+    /**
+     * Heuristic blast radius: `high` when a framework entry point reaches the
+     * symbol, `medium` when at least one caller exists, else `low`.
+     */
+    risk: "low" | "medium" | "high";
+}
+export interface ImpactAnalysis {
+    symbol: SymbolRef;
+    summary: ImpactSummary;
+    callers: ImpactCaller[];
+    entryPoints: {
+        symbol: SymbolRef;
+        entryPointKind: string;
+        detail?: string;
+        viaDepth?: number;
+    }[];
+    routes: WiringEdge[];
+    implementations: ImpactImplementation[];
+    affectedFiles: string[];
+}
+export interface ImpactOptions {
+    maxDepth?: number;
+    includeDynamic?: boolean;
+    limit?: number;
+}
+/**
+ * "What breaks if I change this?" — reverse callers, framework entry points,
+ * routes and implementations in a single response.
+ *
+ * A type aggregates the callers of its contained methods, since call edges bind
+ * to methods and a class is usually changed through its members.
+ */
+export declare function impactAnalysis(store: GraphStore, symbol: string, opts?: ImpactOptions): ImpactAnalysis | null;
+export interface ExplainedMember {
+    symbol: SymbolRef;
+    line?: number;
+    decorators: DecoratorInfo[];
+}
+export interface SymbolExplanation {
+    symbol: SymbolRef;
+    line?: number;
+    isEntryPoint?: boolean;
+    decorators: DecoratorInfo[];
+    members: ExplainedMember[];
+    callers: ImpactCaller[];
+    callees: ImpactCaller[];
+    implementations: ImpactImplementation[];
+    routes: WiringEdge[];
+    injections: WiringEdge[];
+    reflections: WiringEdge[];
+    summary: {
+        members: number;
+        callers: number;
+        callees: number;
+        implementations: number;
+        wiringEdges: number;
+    };
+}
+export interface ExplainOptions {
+    maxDepth?: number;
+    includeDynamic?: boolean;
+    limit?: number;
+}
+/**
+ * One-shot digest of a symbol: what it is, what it is annotated with, what it
+ * contains, who calls it, what it calls, and how frameworks reach it.
+ *
+ * Intended to replace a chain of five tool calls with one round trip.
+ */
+export declare function explainSymbol(store: GraphStore, symbol: string, opts?: ExplainOptions): SymbolExplanation | null;
+export interface CallCycle {
+    size: number;
+    members: SymbolRef[];
+    /** Shortest closed walk through the cycle, starting at the first member. */
+    sample: string[];
+}
+export interface CycleReport {
+    /** Number of multi-node cycles reported (after `minSize`). */
+    cycleCount: number;
+    cycles: CallCycle[];
+    /** Symbols that call themselves directly (size-1 cycles). */
+    selfRecursive: SymbolRef[];
+    edgeKinds: EdgeKind[];
+    scannedNodes: number;
+    scannedEdges: number;
+    /** True when the edge scan hit `maxNodes` and results may be incomplete. */
+    scanTruncated: boolean;
+}
+export interface CycleOptions {
+    edgeKinds?: EdgeKind[];
+    includeDynamic?: boolean;
+    /** Minimum cycle size to report. Default 2 (direct recursion is separate). */
+    minSize?: number;
+    /** Cap on edges scanned, guarding against very large graphs. */
+    maxEdges?: number;
+    limit?: number;
+}
+/**
+ * Find strongly connected components in the call graph with an iterative
+ * Tarjan. A component larger than one node means the members are mutually
+ * reachable, i.e. a call cycle; a self-edge is direct recursion.
+ *
+ * Iterative rather than recursive so deep call chains cannot blow the stack.
+ */
+export declare function findCycles(store: GraphStore, opts?: CycleOptions): CycleReport;
+export type ExportFormat = "mermaid" | "dot";
+export interface GraphExport {
+    format: ExportFormat;
+    diagram: string;
+    nodeCount: number;
+    edgeCount: number;
+    truncated: boolean;
+    /** Symbol the export was centered on, when one was given. */
+    center?: SymbolRef;
+}
+export interface ExportOptions {
+    symbol?: string;
+    radius?: number;
+    edgeKinds?: EdgeKind[];
+    format?: ExportFormat;
+    direction?: "LR" | "TB" | "RL" | "BT";
+    limit?: number;
+}
+/**
+ * Render a neighborhood (or the whole graph) as Mermaid or Graphviz DOT so an
+ * agent can show the user a diagram instead of a list of edges.
+ */
+export declare function exportGraph(store: GraphStore, opts?: ExportOptions): GraphExport | null;