@codragraph/cli 1.6.2

package/dist/core/group/extractors/http-patterns/python.js

@@ -0,0 +1,133 @@
+import Python from 'tree-sitter-python';
+import { compilePatterns, runCompiledPatterns, unquoteLiteral, } from '../tree-sitter-scanner.js';
+/**
+ * Python HTTP plugin. Handles:
+ *   - FastAPI `@app.get("/path")` provider decorators
+ *   - `requests.get/post/...("url")` consumer calls
+ *   - Generic `requests.request("METHOD", "url")` consumer calls
+ */
+const FASTAPI_VERBS = {
+    get: 'GET',
+    post: 'POST',
+    put: 'PUT',
+    delete: 'DELETE',
+    patch: 'PATCH',
+};
+// ─── Provider: FastAPI @app.get/... ──────────────────────────────────
+const FASTAPI_PATTERNS = compilePatterns({
+    name: 'python-fastapi',
+    language: Python,
+    patterns: [
+        {
+            meta: {},
+            query: `
+        (decorator
+          (call
+            function: (attribute
+              object: (identifier) @obj (#eq? @obj "app")
+              attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
+            arguments: (argument_list . (string) @path)))
+      `,
+        },
+    ],
+});
+// ─── Consumer: requests.get/post/... ──────────────────────────────────
+const REQUESTS_VERB_PATTERNS = compilePatterns({
+    name: 'python-requests-verb',
+    language: Python,
+    patterns: [
+        {
+            meta: {},
+            query: `
+        (call
+          function: (attribute
+            object: (identifier) @obj (#eq? @obj "requests")
+            attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
+          arguments: (argument_list . (string) @path))
+      `,
+        },
+    ],
+});
+// ─── Consumer: requests.request("METHOD", "url") ─────────────────────
+const REQUESTS_GENERIC_PATTERNS = compilePatterns({
+    name: 'python-requests-generic',
+    language: Python,
+    patterns: [
+        {
+            meta: {},
+            query: `
+        (call
+          function: (attribute
+            object: (identifier) @obj (#eq? @obj "requests")
+            attribute: (identifier) @method (#eq? @method "request"))
+          arguments: (argument_list . (string) @http_method (string) @path))
+      `,
+        },
+    ],
+});
+export const PYTHON_HTTP_PLUGIN = {
+    name: 'python-http',
+    language: Python,
+    scan(tree) {
+        const out = [];
+        // Providers: FastAPI
+        for (const match of runCompiledPatterns(FASTAPI_PATTERNS, tree)) {
+            const methodNode = match.captures.method;
+            const pathNode = match.captures.path;
+            if (!methodNode || !pathNode)
+                continue;
+            const httpMethod = FASTAPI_VERBS[methodNode.text];
+            if (!httpMethod)
+                continue;
+            const path = unquoteLiteral(pathNode.text);
+            if (path === null)
+                continue;
+            out.push({
+                role: 'provider',
+                framework: 'fastapi',
+                method: httpMethod,
+                path,
+                name: null,
+                confidence: 0.8,
+            });
+        }
+        // Consumers: requests.<verb>
+        for (const match of runCompiledPatterns(REQUESTS_VERB_PATTERNS, tree)) {
+            const methodNode = match.captures.method;
+            const pathNode = match.captures.path;
+            if (!methodNode || !pathNode)
+                continue;
+            const path = unquoteLiteral(pathNode.text);
+            if (path === null)
+                continue;
+            out.push({
+                role: 'consumer',
+                framework: 'python-requests',
+                method: methodNode.text.toUpperCase(),
+                path,
+                name: null,
+                confidence: 0.7,
+            });
+        }
+        // Consumers: requests.request("METHOD", "url")
+        for (const match of runCompiledPatterns(REQUESTS_GENERIC_PATTERNS, tree)) {
+            const methodNode = match.captures.http_method;
+            const pathNode = match.captures.path;
+            if (!methodNode || !pathNode)
+                continue;
+            const methodRaw = unquoteLiteral(methodNode.text);
+            const path = unquoteLiteral(pathNode.text);
+            if (methodRaw === null || path === null)
+                continue;
+            out.push({
+                role: 'consumer',
+                framework: 'python-requests',
+                method: methodRaw.toUpperCase(),
+                path,
+                name: null,
+                confidence: 0.7,
+            });
+        }
+        return out;
+    },
+};

package/dist/core/group/extractors/http-patterns/types.d.ts

@@ -0,0 +1,61 @@
+import type Parser from 'tree-sitter';
+/**
+ * Shared types for the http-route-extractor language plugins.
+ *
+ * Each plugin lives in its own file (java.ts, node.ts, ...) and owns
+ * the tree-sitter grammar import + queries. The top-level
+ * `http-route-extractor.ts` orchestrator only knows about this type
+ * module and the plugin registry (`./index.ts`). It MUST NOT import
+ * any grammar or query text directly — language-specific knowledge
+ * belongs in the plugins.
+ */
+export type HttpRole = 'provider' | 'consumer';
+/**
+ * One raw HTTP detection produced by a plugin's `scan()` function. The
+ * orchestrator converts this into a full `ExtractedContract` by running
+ * path normalization and building the contract id.
+ *
+ * `path` is the raw literal string as it appeared in source (with
+ * `${...}` template placeholders still in place); the orchestrator
+ * runs the appropriate normalizer for provider vs. consumer paths.
+ */
+export interface HttpDetection {
+    role: HttpRole;
+    /** Short framework label, e.g. `'spring'`, `'nest'`, `'express'`. */
+    framework: string;
+    /** HTTP method in upper case (`'GET'`, `'POST'`, ...). */
+    method: string;
+    /** Raw path literal as seen in source (template placeholders intact). */
+    path: string;
+    /**
+     * Symbol name of the handler (for providers) or calling function
+     * (for consumers) when the plugin can determine it structurally.
+     * Null when no good candidate is available.
+     */
+    name: string | null;
+    /** Confidence in (0, 1]. Source-scan plugins typically use 0.7–0.8. */
+    confidence: number;
+}
+/**
+ * One language-scoped HTTP plugin. The plugin owns the tree-sitter
+ * grammar and the `scan` function that translates a parsed tree into
+ * zero or more `HttpDetection`s. Plugins are free to run multiple
+ * compiled pattern bundles internally (see the shared scanner's
+ * `runCompiledPatterns` helper).
+ *
+ * `language` is typed as `unknown` for the same reason as
+ * `LanguagePatterns.language` in `tree-sitter-scanner.ts` — the
+ * grammar modules export different shapes.
+ */
+export interface HttpLanguagePlugin {
+    /** Human-readable plugin name for diagnostics. */
+    name: string;
+    /** tree-sitter grammar object (passed to the shared parser). */
+    language: unknown;
+    /**
+     * Scan a parsed tree and return zero or more HTTP detections. Plugins
+     * must not throw — they should swallow per-match errors so a single
+     * malformed construct does not abort the whole file.
+     */
+    scan(tree: Parser.Tree): HttpDetection[];
+}

package/dist/core/group/extractors/http-patterns/types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/group/extractors/http-route-extractor.d.ts

@@ -0,0 +1,21 @@
+import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
+import type { ExtractedContract, RepoHandle } from '../types.js';
+/**
+ * Canonicalize a provider-side HTTP path for contract-id generation:
+ *   - strip query string
+ *   - lower-case
+ *   - drop trailing slash
+ *   - collapse `:id`, `{id}`, `[id]` path params into a single `{param}`
+ */
+export declare function normalizeHttpPath(p: string): string;
+export declare class HttpRouteExtractor implements ContractExtractor {
+    type: "http";
+    canExtract(_repo: RepoHandle): Promise<boolean>;
+    extract(dbExecutor: CypherExecutor | null, repoPath: string, _repo: RepoHandle): Promise<ExtractedContract[]>;
+    private scanFiles;
+    private extractProvidersGraph;
+    private extractProvidersSourceScan;
+    private extractConsumersGraph;
+    private extractConsumersSourceScan;
+    private dedupeContracts;
+}

package/dist/core/group/extractors/http-route-extractor.js

@@ -0,0 +1,421 @@
+import * as path from 'node:path';
+import { glob } from 'glob';
+import Parser from 'tree-sitter';
+import { readSafe } from './fs-utils.js';
+import { getPluginForFile, HTTP_SCAN_GLOB } from './http-patterns/index.js';
+/**
+ * Language-agnostic orchestrator for HTTP route (provider + consumer)
+ * contract extraction. Two strategies, in order of preference per role:
+ *
+ * 1. **Graph-assisted (Strategy A)** — if a per-repo LadybugDB executor
+ *    is available, read `HANDLES_ROUTE` / `FETCHES` Cypher edges that
+ *    the ingestion pipeline already produced via tree-sitter. This is
+ *    the preferred path because the graph has richer symbol metadata
+ *    (real uids, class/method structure, etc.).
+ *
+ * 2. **Source-scan fallback (Strategy B)** — parse files directly with
+ *    the per-language plugin registry in `./http-patterns/`. Used when
+ *    the graph has no routes/fetches for this repo (e.g. a repo that
+ *    hasn't been indexed yet, or whose indexer doesn't know the
+ *    framework). Each plugin owns its tree-sitter grammar and query
+ *    sources — this orchestrator imports NO grammars or query strings.
+ *
+ * Adding a new language for Strategy B is a one-file edit in
+ * `http-patterns/index.ts`: register a new `HttpLanguagePlugin` and
+ * widen `HTTP_SCAN_GLOB` if needed.
+ */
+// ─── Graph-assisted queries ──────────────────────────────────────────
+const HANDLES_ROUTE_QUERY = `
+MATCH (handlerFile:File)-[r:CodeRelation {type: 'HANDLES_ROUTE'}]->(route:Route)
+RETURN handlerFile.id AS fileId, handlerFile.filePath AS filePath,
+       route.name AS routePath, route.id AS routeId,
+       route.responseKeys AS responseKeys,
+       r.reason AS routeSource`;
+const FETCHES_QUERY = `
+MATCH (callerFile:File)-[r:CodeRelation {type: 'FETCHES'}]->(route:Route)
+RETURN callerFile.id AS fileId, callerFile.filePath AS filePath,
+       route.name AS routePath, route.id AS routeId,
+       r.reason AS fetchReason`;
+const CONTAINS_QUERY = `
+MATCH (file:File {id: $fileId})<-[:CodeRelation {type: 'CONTAINS'}]-(sym)
+WHERE sym.startLine IS NOT NULL
+RETURN sym.id AS uid, sym.name AS name, sym.filePath AS filePath, labels(sym) AS labels
+ORDER BY sym.startLine`;
+// ─── Path normalization (shared between provider / consumer paths) ──
+/**
+ * Canonicalize a provider-side HTTP path for contract-id generation:
+ *   - strip query string
+ *   - lower-case
+ *   - drop trailing slash
+ *   - collapse `:id`, `{id}`, `[id]` path params into a single `{param}`
+ */
+export function normalizeHttpPath(p) {
+    let s = p.trim().split('?')[0].toLowerCase().replace(/\/+$/, '');
+    s = s.replace(/:\w+/g, '{param}');
+    s = s.replace(/\{[^}]+\}/g, '{param}');
+    s = s.replace(/\[[^\]]+\]/g, '{param}');
+    // Preserve root: after stripping trailing slashes, the root "/"
+    // collapses to "" which would produce malformed contract ids like
+    // `http::GET::`. Restore a single slash for the root case.
+    return s === '' ? '/' : s;
+}
+/**
+ * Consumer-side normalization is more aggressive:
+ *   - template literals (`${x}`) → `{param}`
+ *   - strip protocol + host if the URL is absolute
+ *   - numeric segments → `{param}` (so `/api/orders/42` → `/api/orders/{param}`)
+ */
+function normalizeConsumerPath(url) {
+    const templated = url.replace(/\$\{[^}]+\}/g, '{param}').trim();
+    let pathOnly = templated;
+    if (/^https?:\/\//i.test(templated)) {
+        try {
+            pathOnly = new URL(templated).pathname;
+        }
+        catch {
+            pathOnly = templated.replace(/^https?:\/\/[^/]+/i, '');
+        }
+    }
+    const normalized = normalizeHttpPath(pathOnly || '/');
+    const segments = normalized
+        .split('/')
+        .filter(Boolean)
+        .map((segment) => (/^\d+$/.test(segment) ? '{param}' : segment));
+    return `/${segments.join('/')}`.replace(/\/+$/, '') || '/';
+}
+function contractIdFor(method, pathNorm) {
+    return `http::${method.toUpperCase()}::${pathNorm}`;
+}
+// ─── Graph row helpers ───────────────────────────────────────────────
+function methodFromRouteReason(reason) {
+    const r = reason || '';
+    if (/GetMapping|decorator-Get/i.test(r))
+        return 'GET';
+    if (/PostMapping|decorator-Post/i.test(r))
+        return 'POST';
+    if (/PutMapping|decorator-Put/i.test(r))
+        return 'PUT';
+    if (/DeleteMapping|decorator-Delete/i.test(r))
+        return 'DELETE';
+    if (/PatchMapping|decorator-Patch/i.test(r))
+        return 'PATCH';
+    return null;
+}
+function pickSymbolUid(rows, preferredName) {
+    const norm = (x) => String(x ?? '');
+    const labeled = rows.filter((r) => {
+        const labels = r.labels ?? r[3];
+        const s = JSON.stringify(labels);
+        return s.includes('Method') || s.includes('Function');
+    });
+    const pool = labeled.length > 0 ? labeled : rows;
+    if (preferredName) {
+        const hit = pool.find((r) => norm(r.name ?? r[1]) === preferredName);
+        if (hit) {
+            return {
+                uid: norm(hit.uid ?? hit[0]),
+                name: norm(hit.name ?? hit[1]),
+                filePath: norm(hit.filePath ?? hit[2]),
+            };
+        }
+    }
+    const first = pool[0] || rows[0];
+    return {
+        uid: norm(first?.uid ?? first?.[0]),
+        name: norm(first?.name ?? first?.[1]),
+        filePath: norm(first?.filePath ?? first?.[2]),
+    };
+}
+// ─── Orchestrator ────────────────────────────────────────────────────
+export class HttpRouteExtractor {
+    type = 'http';
+    async canExtract(_repo) {
+        return true;
+    }
+    async extract(dbExecutor, repoPath, _repo) {
+        // Parse each file at most once and reuse the plugin results across
+        // both graph-assisted enrichment and source-scan emission.
+        const parser = new Parser();
+        const cachedDetections = new Map();
+        const getDetections = (rel) => {
+            const cached = cachedDetections.get(rel);
+            if (cached)
+                return cached;
+            const plugin = getPluginForFile(rel);
+            if (!plugin) {
+                cachedDetections.set(rel, []);
+                return [];
+            }
+            const content = readSafe(repoPath, rel);
+            if (!content) {
+                cachedDetections.set(rel, []);
+                return [];
+            }
+            try {
+                parser.setLanguage(plugin.language);
+                const tree = parser.parse(content);
+                const detections = plugin.scan(tree);
+                cachedDetections.set(rel, detections);
+                return detections;
+            }
+            catch {
+                cachedDetections.set(rel, []);
+                return [];
+            }
+        };
+        // Glob the source-scan file list at most once per extract() —
+        // both provider and consumer fallback paths share the same list.
+        let scannedFiles = null;
+        const getScannedFiles = async () => {
+            if (scannedFiles)
+                return scannedFiles;
+            scannedFiles = await this.scanFiles(repoPath);
+            return scannedFiles;
+        };
+        const graphProviders = dbExecutor != null ? await this.extractProvidersGraph(dbExecutor, getDetections) : [];
+        const providers = graphProviders.length > 0
+            ? graphProviders
+            : this.extractProvidersSourceScan(await getScannedFiles(), getDetections);
+        const graphConsumers = dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, getDetections) : [];
+        const consumers = graphConsumers.length > 0
+            ? graphConsumers
+            : this.extractConsumersSourceScan(await getScannedFiles(), getDetections);
+        return [...providers, ...consumers];
+    }
+    async scanFiles(repoPath) {
+        return glob(HTTP_SCAN_GLOB, {
+            cwd: repoPath,
+            ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**', '**/vendor/**'],
+            nodir: true,
+        });
+    }
+    // ─── Graph-assisted providers ──────────────────────────────────────
+    async extractProvidersGraph(db, getDetections) {
+        const out = [];
+        let rows;
+        try {
+            rows = await db(HANDLES_ROUTE_QUERY);
+        }
+        catch {
+            return [];
+        }
+        for (const row of rows) {
+            const filePath = String(row.filePath ?? '');
+            const routePath = String(row.routePath ?? '');
+            const routeSource = String(row.routeSource ?? row.routeReason ?? '');
+            let method = methodFromRouteReason(routeSource);
+            // Look up handler name (and backfill method if missing) from the
+            // plugin's scan of the handler file. This replaces the old
+            // regex-based `inferMethodFromFileScan` and `pickJavaHandlerName`
+            // helpers — tree-sitter gives both pieces of information
+            // structurally. Always run the lookup: even when method is set by
+            // `methodFromRouteReason`, we still need the handler name.
+            const detections = filePath ? getDetections(filePath) : [];
+            const providerDetections = detections.filter((d) => d.role === 'provider');
+            let handlerName = null;
+            const normalizedRoute = normalizeHttpPath(routePath);
+            // Candidates share the same normalized path. When multiple
+            // detections at the same path exist (e.g. GET + POST /api/orders
+            // in one router), a blind `.find()` silently returned the first
+            // verb — attaching the wrong handler and, when method was not
+            // already pinned by the route reason, the wrong method too.
+            // Disambiguate by method when we know it; refuse to guess when
+            // we don't.
+            const candidates = providerDetections.filter((d) => normalizeHttpPath(d.path) === normalizedRoute);
+            let match;
+            const ambiguousCandidates = !method && candidates.length > 1;
+            if (method) {
+                match = candidates.find((d) => d.method === method);
+            }
+            else if (candidates.length === 1) {
+                match = candidates[0];
+            }
+            // else: multiple candidates + unknown method → leave match
+            // undefined so handlerName stays null and skip symbol
+            // enrichment below, keeping the file-basename fallback instead
+            // of letting pickSymbolUid silently pick the first Function /
+            // Method in the file (which reintroduces the mis-attribution
+            // we were trying to avoid). Method stays at the conservative
+            // 'GET' default set below.
+            if (match) {
+                if (!method)
+                    method = match.method;
+                handlerName = match.name;
+            }
+            if (!method)
+                method = 'GET';
+            const pathNorm = normalizeHttpPath(routePath);
+            const cid = contractIdFor(method, pathNorm);
+            let symbolUid = '';
+            let symbolName = path.basename(filePath) || 'handler';
+            let symPath = filePath;
+            const fileId = row.fileId ?? row[0];
+            if (fileId && !ambiguousCandidates) {
+                try {
+                    const syms = await db(CONTAINS_QUERY, { fileId });
+                    if (syms.length > 0) {
+                        const picked = pickSymbolUid(syms, handlerName);
+                        symbolUid = picked.uid;
+                        symbolName = picked.name;
+                        symPath = picked.filePath || filePath;
+                    }
+                }
+                catch {
+                    /* ignore */
+                }
+            }
+            out.push({
+                contractId: cid,
+                type: 'http',
+                role: 'provider',
+                symbolUid,
+                symbolRef: { filePath: symPath, name: symbolName },
+                symbolName,
+                confidence: 0.9,
+                meta: {
+                    method,
+                    path: pathNorm,
+                    pathSegments: pathNorm.split('/').filter(Boolean),
+                    extractionStrategy: 'graph_assisted',
+                    routeSource,
+                },
+            });
+        }
+        return out;
+    }
+    // ─── Source-scan providers ─────────────────────────────────────────
+    extractProvidersSourceScan(files, getDetections) {
+        const out = [];
+        for (const rel of files) {
+            const detections = getDetections(rel);
+            for (const d of detections) {
+                if (d.role !== 'provider')
+                    continue;
+                const pathNorm = normalizeHttpPath(d.path);
+                out.push({
+                    contractId: contractIdFor(d.method, pathNorm),
+                    type: 'http',
+                    role: 'provider',
+                    symbolUid: '',
+                    symbolRef: { filePath: rel, name: d.name ?? 'handler' },
+                    symbolName: d.name ?? 'handler',
+                    confidence: d.confidence,
+                    meta: {
+                        method: d.method,
+                        path: pathNorm,
+                        pathSegments: pathNorm.split('/').filter(Boolean),
+                        extractionStrategy: 'source_scan',
+                        framework: d.framework,
+                    },
+                });
+            }
+        }
+        return this.dedupeContracts(out);
+    }
+    // ─── Graph-assisted consumers ──────────────────────────────────────
+    async extractConsumersGraph(db, getDetections) {
+        const out = [];
+        let rows;
+        try {
+            rows = await db(FETCHES_QUERY);
+        }
+        catch {
+            return [];
+        }
+        for (const row of rows) {
+            const filePath = String(row.filePath ?? '');
+            const routePath = String(row.routePath ?? '');
+            const pathNorm = normalizeHttpPath(routePath);
+            let method = 'GET';
+            // Prefer the plugin's detected method if we can find a matching
+            // fetch/axios call in the same file.
+            const detections = filePath ? getDetections(filePath) : [];
+            // Symmetric to the provider path: if multiple consumer calls in
+            // the same file share the same normalized path (e.g. a GET
+            // fetch AND a POST fetch to `/api/orders`), `.find()` silently
+            // picked the first verb and keyed the contract id on the wrong
+            // method. With no upstream method signal here, refuse to guess
+            // when candidates are ambiguous — leave `method` at its
+            // conservative 'GET' default.
+            const consumerCandidates = detections.filter((d) => d.role === 'consumer' && normalizeConsumerPath(d.path) === pathNorm);
+            if (consumerCandidates.length === 1) {
+                method = consumerCandidates[0].method;
+            }
+            const cid = contractIdFor(method, pathNorm);
+            let symbolUid = '';
+            let symbolName = 'fetch';
+            let symPath = filePath;
+            const fileId = row.fileId ?? row[0];
+            if (fileId) {
+                try {
+                    const syms = await db(CONTAINS_QUERY, { fileId });
+                    if (syms.length > 0) {
+                        const picked = pickSymbolUid(syms, null);
+                        symbolUid = picked.uid;
+                        symbolName = picked.name;
+                        symPath = picked.filePath || filePath;
+                    }
+                }
+                catch {
+                    /* ignore */
+                }
+            }
+            out.push({
+                contractId: cid,
+                type: 'http',
+                role: 'consumer',
+                symbolUid,
+                symbolRef: { filePath: symPath, name: symbolName },
+                symbolName,
+                confidence: 0.9,
+                meta: {
+                    method,
+                    path: pathNorm,
+                    extractionStrategy: 'graph_assisted',
+                    fetchReason: String(row.fetchReason ?? ''),
+                },
+            });
+        }
+        return out;
+    }
+    // ─── Source-scan consumers ─────────────────────────────────────────
+    extractConsumersSourceScan(files, getDetections) {
+        const out = [];
+        for (const rel of files) {
+            const detections = getDetections(rel);
+            for (const d of detections) {
+                if (d.role !== 'consumer')
+                    continue;
+                const pathNorm = normalizeConsumerPath(d.path);
+                out.push({
+                    contractId: contractIdFor(d.method, pathNorm),
+                    type: 'http',
+                    role: 'consumer',
+                    symbolUid: '',
+                    symbolRef: { filePath: rel, name: 'fetch' },
+                    symbolName: 'fetch',
+                    confidence: d.confidence,
+                    meta: {
+                        method: d.method,
+                        path: pathNorm,
+                        extractionStrategy: 'source_scan',
+                        framework: d.framework,
+                    },
+                });
+            }
+        }
+        return this.dedupeContracts(out);
+    }
+    dedupeContracts(items) {
+        const seen = new Set();
+        const out = [];
+        for (const c of items) {
+            const k = `${c.contractId}|${c.symbolRef.filePath}|${c.symbolRef.name}`;
+            if (seen.has(k))
+                continue;
+            seen.add(k);
+            out.push(c);
+        }
+        return out;
+    }
+}