codesight 1.3.2 → 1.5.0

package/dist/ast/extract-python.js

@@ -0,0 +1,388 @@
+/**
+ * Python AST extraction via subprocess.
+ * Spawns python3 with an inline script using stdlib `ast` module.
+ * Zero dependencies — if the project uses Python, the interpreter is there.
+ *
+ * Extracts:
+ * - FastAPI/Flask route decorators with precise path + method
+ * - Django urlpatterns with path() calls
+ * - SQLAlchemy model classes with Column types, flags, and relationships
+ */
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+const execFileP = promisify(execFile);
+// The Python script that does AST parsing via stdlib
+// Outputs JSON to stdout
+const PYTHON_ROUTE_SCRIPT = `
+import ast, json, sys
+
+def extract_routes(source, filename):
+    try:
+        tree = ast.parse(source, filename)
+    except SyntaxError:
+        return []
+
+    routes = []
+
+    for node in ast.walk(tree):
+        # FastAPI/Flask style: @router.get("/path") or @app.route("/path", methods=["GET","POST"])
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            for dec in node.decorator_list:
+                route = parse_route_decorator(dec)
+                if route:
+                    for r in route:
+                        routes.append(r)
+
+        # Django: path("url", view) in urlpatterns list
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                if isinstance(target, ast.Name) and target.id == 'urlpatterns':
+                    if isinstance(node.value, (ast.List, ast.BinOp)):
+                        routes.extend(parse_urlpatterns(node.value))
+
+    return routes
+
+def parse_route_decorator(dec):
+    results = []
+
+    # @router.get("/path") / @app.post("/path") — Attribute call
+    if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute):
+        method_name = dec.func.attr.upper()
+        methods_map = {'GET':'GET','POST':'POST','PUT':'PUT','PATCH':'PATCH',
+                       'DELETE':'DELETE','OPTIONS':'OPTIONS','HEAD':'HEAD'}
+
+        if method_name in methods_map and dec.args:
+            path = get_str(dec.args[0])
+            if path is not None:
+                results.append({'method': methods_map[method_name], 'path': path})
+
+        # @app.route("/path", methods=["GET","POST"])
+        if dec.func.attr == 'route' and dec.args:
+            path = get_str(dec.args[0])
+            if path is not None:
+                methods = ['GET']
+                for kw in dec.keywords:
+                    if kw.arg == 'methods' and isinstance(kw.value, ast.List):
+                        methods = [get_str(e) for e in kw.value.elts if get_str(e)]
+                for m in methods:
+                    results.append({'method': m.upper(), 'path': path})
+
+    # @app.api_route("/path", methods=["GET","POST"]) — FastAPI api_route
+    if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute):
+        if dec.func.attr == 'api_route' and dec.args:
+            path = get_str(dec.args[0])
+            if path is not None:
+                methods = ['GET']
+                for kw in dec.keywords:
+                    if kw.arg == 'methods' and isinstance(kw.value, ast.List):
+                        methods = [get_str(e) for e in kw.value.elts if get_str(e)]
+                for m in methods:
+                    results.append({'method': m.upper(), 'path': path})
+
+    return results if results else None
+
+def parse_urlpatterns(node):
+    routes = []
+    elements = []
+    if isinstance(node, ast.List):
+        elements = node.elts
+    elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
+        # urlpatterns = [...] + [...]
+        if isinstance(node.left, ast.List):
+            elements.extend(node.left.elts)
+        if isinstance(node.right, ast.List):
+            elements.extend(node.right.elts)
+
+    for elt in elements:
+        if isinstance(elt, ast.Call):
+            func_name = ''
+            if isinstance(elt.func, ast.Name):
+                func_name = elt.func.id
+            elif isinstance(elt.func, ast.Attribute):
+                func_name = elt.func.attr
+
+            if func_name in ('path', 're_path', 'url') and elt.args:
+                path_str = get_str(elt.args[0])
+                if path_str is not None:
+                    routes.append({'method': 'ALL', 'path': '/' + path_str.lstrip('/')})
+
+    return routes
+
+def get_str(node):
+    if isinstance(node, ast.Constant) and isinstance(node.value, str):
+        return node.value
+    if isinstance(node, ast.Str):  # Python 3.7
+        return node.s
+    return None
+
+source = sys.stdin.read()
+filename = sys.argv[1] if len(sys.argv) > 1 else '<stdin>'
+result = extract_routes(source, filename)
+print(json.dumps(result))
+`;
+const PYTHON_SCHEMA_SCRIPT = `
+import ast, json, sys
+
+def extract_models(source, filename):
+    try:
+        tree = ast.parse(source, filename)
+    except SyntaxError:
+        return []
+
+    models = []
+
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.ClassDef):
+            continue
+
+        # Check if inherits from Base, Model, DeclarativeBase, db.Model
+        is_model = False
+        for base in node.bases:
+            name = ''
+            if isinstance(base, ast.Name):
+                name = base.id
+            elif isinstance(base, ast.Attribute):
+                name = base.attr
+            if name in ('Base', 'Model', 'DeclarativeBase', 'AbstractBase'):
+                is_model = True
+                break
+        if not is_model:
+            continue
+
+        fields = []
+        relations = []
+
+        for item in node.body:
+            # Column assignments: field = Column(Type, ...) or field: Mapped[Type] = mapped_column(...)
+            if isinstance(item, ast.Assign):
+                for target in item.targets:
+                    if isinstance(target, ast.Name):
+                        field_info = parse_column_or_rel(target.id, item.value)
+                        if field_info:
+                            if field_info['kind'] == 'field':
+                                fields.append(field_info)
+                            else:
+                                relations.append(field_info)
+
+            if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):
+                field_name = item.target.id
+                if field_name.startswith('__'):
+                    continue
+                if item.value:
+                    field_info = parse_column_or_rel(field_name, item.value)
+                    if field_info:
+                        if field_info['kind'] == 'field':
+                            # Try to get type from annotation
+                            ann_type = extract_mapped_type(item.annotation)
+                            if ann_type and field_info['type'] == 'unknown':
+                                field_info['type'] = ann_type
+                            fields.append(field_info)
+                        else:
+                            relations.append(field_info)
+
+        if fields:
+            models.append({
+                'name': node.name,
+                'fields': fields,
+                'relations': relations,
+            })
+
+    return models
+
+def parse_column_or_rel(name, value):
+    if name.startswith('__'):
+        return None
+
+    # relationship("Model")
+    if isinstance(value, ast.Call):
+        func_name = get_func_name(value)
+
+        if func_name in ('relationship', 'db.relationship'):
+            target = ''
+            if value.args:
+                target = get_str(value.args[0]) or ''
+            return {'kind': 'relation', 'name': name, 'target': target}
+
+        if func_name in ('Column', 'db.Column', 'mapped_column'):
+            col_type = 'unknown'
+            flags = []
+
+            for arg in value.args:
+                t = get_type_name(arg)
+                if t:
+                    col_type = t
+                if isinstance(arg, ast.Call) and get_func_name(arg) == 'ForeignKey':
+                    flags.append('fk')
+
+            for kw in value.keywords:
+                if kw.arg == 'primary_key' and is_true(kw.value):
+                    flags.append('pk')
+                elif kw.arg == 'unique' and is_true(kw.value):
+                    flags.append('unique')
+                elif kw.arg == 'nullable' and is_true(kw.value):
+                    flags.append('nullable')
+                elif kw.arg == 'default':
+                    flags.append('default')
+                elif kw.arg == 'index' and is_true(kw.value):
+                    flags.append('index')
+
+            return {'kind': 'field', 'name': name, 'type': col_type, 'flags': flags}
+
+    return None
+
+def extract_mapped_type(annotation):
+    # Mapped[int] or Mapped[Optional[str]]
+    if isinstance(annotation, ast.Subscript):
+        if isinstance(annotation.value, ast.Name) and annotation.value.id == 'Mapped':
+            slice_node = annotation.slice
+            if isinstance(slice_node, ast.Name):
+                return slice_node.id
+            if isinstance(slice_node, ast.Subscript) and isinstance(slice_node.value, ast.Name):
+                if slice_node.value.id == 'Optional':
+                    inner = slice_node.slice
+                    if isinstance(inner, ast.Name):
+                        return inner.id
+    return None
+
+def get_func_name(node):
+    if isinstance(node.func, ast.Name):
+        return node.func.id
+    if isinstance(node.func, ast.Attribute):
+        if isinstance(node.func.value, ast.Name):
+            return node.func.value.id + '.' + node.func.attr
+        return node.func.attr
+    return ''
+
+def get_type_name(node):
+    if isinstance(node, ast.Name):
+        known = {'String','Integer','Boolean','Float','Text','DateTime','JSON',
+                 'UUID','BigInteger','SmallInteger','Numeric','Date','Time',
+                 'LargeBinary','Enum','ARRAY','JSONB'}
+        if node.id in known:
+            return node.id
+    if isinstance(node, ast.Call):
+        return get_type_name(node.func)
+    if isinstance(node, ast.Attribute):
+        return node.attr
+    return None
+
+def get_str(node):
+    if isinstance(node, ast.Constant) and isinstance(node.value, str):
+        return node.value
+    return None
+
+def is_true(node):
+    if isinstance(node, ast.Constant):
+        return node.value is True
+    if isinstance(node, ast.NameConstant):  # Python 3.7
+        return node.value is True
+    return False
+
+source = sys.stdin.read()
+filename = sys.argv[1] if len(sys.argv) > 1 else '<stdin>'
+result = extract_models(source, filename)
+print(json.dumps(result))
+`;
+let pythonAvailable = null;
+let pythonCmd = "python3";
+async function findPython() {
+    if (pythonAvailable !== null)
+        return pythonAvailable;
+    for (const cmd of ["python3", "python"]) {
+        try {
+            const { stdout } = await execFileP(cmd, ["--version"], { timeout: 3000 });
+            if (stdout.includes("Python 3")) {
+                pythonCmd = cmd;
+                pythonAvailable = true;
+                return true;
+            }
+        }
+        catch { }
+    }
+    pythonAvailable = false;
+    return false;
+}
+import { spawn } from "node:child_process";
+async function runPythonWithStdin(script, source, filename) {
+    if (!(await findPython()))
+        return null;
+    return new Promise((resolve) => {
+        const proc = spawn(pythonCmd, ["-c", script, filename], {
+            timeout: 10000,
+            env: { ...process.env, PYTHONDONTWRITEBYTECODE: "1" },
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+        let stdout = "";
+        let stderr = "";
+        proc.stdout.on("data", (data) => { stdout += data.toString(); });
+        proc.stderr.on("data", (data) => { stderr += data.toString(); });
+        proc.on("close", (code) => {
+            if (code !== 0 || !stdout.trim()) {
+                resolve(null);
+                return;
+            }
+            try {
+                resolve(JSON.parse(stdout.trim()));
+            }
+            catch {
+                resolve(null);
+            }
+        });
+        proc.on("error", () => resolve(null));
+        proc.stdin.write(source);
+        proc.stdin.end();
+    });
+}
+/**
+ * Extract routes from a Python file using AST.
+ * Returns routes with confidence: "ast", or null if Python is unavailable.
+ */
+export async function extractPythonRoutesAST(filePath, content, framework, tags) {
+    const result = await runPythonWithStdin(PYTHON_ROUTE_SCRIPT, content, filePath);
+    if (!result || !Array.isArray(result) || result.length === 0)
+        return null;
+    return result.map((r) => ({
+        method: r.method,
+        path: r.path,
+        file: filePath,
+        tags,
+        framework,
+        params: extractPathParams(r.path),
+        confidence: "ast",
+    }));
+}
+/**
+ * Extract SQLAlchemy models from a Python file using AST.
+ * Returns models with confidence: "ast", or null if Python is unavailable.
+ */
+export async function extractSQLAlchemyAST(filePath, content) {
+    const result = await runPythonWithStdin(PYTHON_SCHEMA_SCRIPT, content, filePath);
+    if (!result || !Array.isArray(result) || result.length === 0)
+        return null;
+    return result.map((m) => ({
+        name: m.name,
+        fields: (m.fields || []).map((f) => ({
+            name: f.name,
+            type: f.type || "unknown",
+            flags: f.flags || [],
+        })),
+        relations: (m.relations || []).map((r) => `${r.name}: ${r.target}`),
+        orm: "sqlalchemy",
+        confidence: "ast",
+    }));
+}
+/**
+ * Check if Python 3 is available on this system.
+ */
+export async function isPythonAvailable() {
+    return findPython();
+}
+function extractPathParams(path) {
+    const params = [];
+    // FastAPI: {param} / Flask: <param> / Django: <type:param>
+    const regex = /[{<](?:\w+:)?(\w+)[}>]/g;
+    let m;
+    while ((m = regex.exec(path)) !== null)
+        params.push(m[1]);
+    return params;
+}

package/dist/config.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Configuration loader: reads codesight.config.(ts|js|json) from project root.
+ */
+import type { CodesightConfig } from "./types.js";
+/**
+ * Load config from project root. Returns empty config if no config file found.
+ */
+export declare function loadConfig(root: string): Promise<CodesightConfig>;
+/**
+ * Merges CLI args with config file values (CLI takes precedence).
+ */
+export declare function mergeCliConfig(config: CodesightConfig, cli: {
+    maxDepth?: number;
+    outputDir?: string;
+    profile?: string;
+}): CodesightConfig;

package/dist/config.js

@@ -0,0 +1,96 @@
+/**
+ * Configuration loader: reads codesight.config.(ts|js|json) from project root.
+ */
+import { readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+const CONFIG_FILES = [
+    "codesight.config.ts",
+    "codesight.config.js",
+    "codesight.config.mjs",
+    "codesight.config.json",
+];
+async function fileExists(path) {
+    try {
+        await stat(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Load config from project root. Returns empty config if no config file found.
+ */
+export async function loadConfig(root) {
+    for (const filename of CONFIG_FILES) {
+        const configPath = join(root, filename);
+        if (!(await fileExists(configPath)))
+            continue;
+        try {
+            if (filename.endsWith(".json")) {
+                const content = await readFile(configPath, "utf-8");
+                return JSON.parse(content);
+            }
+            if (filename.endsWith(".ts")) {
+                // Try loading with tsx or ts-node if available
+                return await loadTsConfig(configPath, root);
+            }
+            // JS/MJS — dynamic import
+            const module = await import(pathToFileURL(configPath).href);
+            return (module.default || module);
+        }
+        catch (err) {
+            console.warn(`  Warning: failed to load ${filename}: ${err.message}`);
+            return {};
+        }
+    }
+    // Also check package.json "codesight" field
+    try {
+        const pkgPath = join(root, "package.json");
+        if (await fileExists(pkgPath)) {
+            const pkg = JSON.parse(await readFile(pkgPath, "utf-8"));
+            if (pkg.codesight && typeof pkg.codesight === "object") {
+                return pkg.codesight;
+            }
+        }
+    }
+    catch { }
+    return {};
+}
+async function loadTsConfig(configPath, _root) {
+    // Strategy 1: try tsx via dynamic import of the .ts file directly
+    // (works if tsx or ts-node is installed)
+    try {
+        const module = await import(pathToFileURL(configPath).href);
+        return (module.default || module);
+    }
+    catch { }
+    // Strategy 2: read as text and extract JSON-like config
+    // (fallback for when no TS loader is available)
+    const content = await readFile(configPath, "utf-8");
+    // Try to extract the config object from simple export default { ... }
+    const match = content.match(/export\s+default\s+({[\s\S]*})\s*;?\s*$/m);
+    if (match) {
+        try {
+            // Use Function constructor to evaluate the object literal
+            // Safe here since this is user's own config file in their project
+            const fn = new Function(`return (${match[1]})`);
+            return fn();
+        }
+        catch { }
+    }
+    console.warn(`  Warning: cannot load codesight.config.ts (install tsx for TS config support)`);
+    return {};
+}
+/**
+ * Merges CLI args with config file values (CLI takes precedence).
+ */
+export function mergeCliConfig(config, cli) {
+    return {
+        ...config,
+        maxDepth: cli.maxDepth ?? config.maxDepth,
+        outputDir: cli.outputDir ?? config.outputDir,
+        profile: cli.profile ?? config.profile,
+    };
+}

package/dist/detectors/routes.js

@@ -2,6 +2,8 @@ import { relative, basename } from "node:path";
 import { readFileSafe } from "../scanner.js";
 import { loadTypeScript } from "../ast/loader.js";
 import { extractRoutesAST } from "../ast/extract-routes.js";
+import { extractPythonRoutesAST } from "../ast/extract-python.js";
+import { extractGoRoutesStructured } from "../ast/extract-go.js";
 const HTTP_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"];
 const TAG_PATTERNS = [
     ["auth", [/auth/i, /jwt/i, /token/i, /session/i, /bearer/i, /passport/i, /clerk/i, /betterAuth/i, /better-auth/i]],
@@ -581,6 +583,14 @@ async function detectFastAPIRoutes(files, project) {
         if (!content.includes("fastapi") && !content.includes("FastAPI") && !content.includes("APIRouter"))
             continue;
         const rel = relative(project.root, file);
+        const tags = detectTags(content);
+        // Try Python AST first
+        const astRoutes = await extractPythonRoutesAST(rel, content, "fastapi", tags);
+        if (astRoutes && astRoutes.length > 0) {
+            routes.push(...astRoutes);
+            continue;
+        }
+        // Fallback to regex
         const routePattern = /@\w+\s*\.\s*(get|post|put|patch|delete|options)\s*\(\s*['"]([^'"]+)['"]/gi;
         let match;
         while ((match = routePattern.exec(content)) !== null) {
@@ -588,7 +598,7 @@ async function detectFastAPIRoutes(files, project) {
                 method: match[1].toUpperCase(),
                 path: match[2],
                 file: rel,
-                tags: detectTags(content),
+                tags,
                 framework: "fastapi",
             });
         }
@@ -604,6 +614,14 @@ async function detectFlaskRoutes(files, project) {
         if (!content.includes("flask") && !content.includes("Flask") && !content.includes("Blueprint"))
             continue;
         const rel = relative(project.root, file);
+        const tags = detectTags(content);
+        // Try Python AST first
+        const astRoutes = await extractPythonRoutesAST(rel, content, "flask", tags);
+        if (astRoutes && astRoutes.length > 0) {
+            routes.push(...astRoutes);
+            continue;
+        }
+        // Fallback to regex
         const routePattern = /@(?:app|bp|blueprint|\w+)\s*\.\s*route\s*\(\s*['"]([^'"]+)['"](?:\s*,\s*methods\s*=\s*\[([^\]]+)\])?\s*\)/gi;
         let match;
         while ((match = routePattern.exec(content)) !== null) {
@@ -616,7 +634,7 @@ async function detectFlaskRoutes(files, project) {
                     method,
                     path,
                     file: rel,
-                    tags: detectTags(content),
+                    tags,
                     framework: "flask",
                 });
             }
@@ -631,7 +649,14 @@ async function detectDjangoRoutes(files, project) {
     for (const file of pyFiles) {
         const content = await readFileSafe(file);
         const rel = relative(project.root, file);
-        // path("api/v1/users/", views.UserView.as_view())
+        const tags = detectTags(content);
+        // Try Python AST first
+        const astRoutes = await extractPythonRoutesAST(rel, content, "django", tags);
+        if (astRoutes && astRoutes.length > 0) {
+            routes.push(...astRoutes);
+            continue;
+        }
+        // Fallback to regex
         const pathPattern = /path\s*\(\s*['"]([^'"]*)['"]\s*,/g;
         let match;
         while ((match = pathPattern.exec(content)) !== null) {
@@ -639,7 +664,7 @@ async function detectDjangoRoutes(files, project) {
                 method: "ALL",
                 path: "/" + match[1],
                 file: rel,
-                tags: detectTags(content),
+                tags,
                 framework: "django",
             });
         }
@@ -653,72 +678,43 @@ async function detectGoRoutes(files, project, fw) {
     for (const file of goFiles) {
         const content = await readFileSafe(file);
         const rel = relative(project.root, file);
-        if (fw === "gin") {
-            const pattern = /\.\s*(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)\s*\(\s*["']([^"']+)["']/g;
-            let match;
-            while ((match = pattern.exec(content)) !== null) {
-                routes.push({
-                    method: match[1],
-                    path: match[2],
-                    file: rel,
-                    tags: detectTags(content),
-                    framework: fw,
-                });
-            }
-        }
-        else if (fw === "fiber") {
-            const pattern = /\.\s*(Get|Post|Put|Patch|Delete|Options|Head)\s*\(\s*["']([^"']+)["']/g;
-            let match;
-            while ((match = pattern.exec(content)) !== null) {
-                routes.push({
-                    method: match[1].toUpperCase(),
-                    path: match[2],
-                    file: rel,
-                    tags: detectTags(content),
-                    framework: fw,
-                });
-            }
+        const tags = detectTags(content);
+        // Use structured parser (brace-tracking + group prefix resolution)
+        const structuredRoutes = extractGoRoutesStructured(rel, content, fw, tags);
+        if (structuredRoutes.length > 0) {
+            routes.push(...structuredRoutes);
+            continue;
         }
-        else if (fw === "echo") {
-            // e.GET("/path", handler) or g.POST("/path", handler)
+        // Fallback to simple regex for files where structured parser found nothing
+        if (fw === "gin" || fw === "echo") {
             const pattern = /\.\s*(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)\s*\(\s*["']([^"']+)["']/g;
             let match;
             while ((match = pattern.exec(content)) !== null) {
-                routes.push({
-                    method: match[1],
-                    path: match[2],
-                    file: rel,
-                    tags: detectTags(content),
-                    framework: fw,
-                });
+                routes.push({ method: match[1], path: match[2], file: rel, tags, framework: fw });
             }
         }
-        else if (fw === "chi") {
-            // r.Get("/path", handler), r.Post("/path", handler)
+        else if (fw === "fiber" || fw === "chi") {
             const pattern = /\.\s*(Get|Post|Put|Patch|Delete|Options|Head)\s*\(\s*["']([^"']+)["']/g;
             let match;
             while ((match = pattern.exec(content)) !== null) {
-                routes.push({
-                    method: match[1].toUpperCase(),
-                    path: match[2],
-                    file: rel,
-                    tags: detectTags(content),
-                    framework: fw,
-                });
+                routes.push({ method: match[1].toUpperCase(), path: match[2], file: rel, tags, framework: fw });
             }
         }
         else {
-            // net/http: http.HandleFunc("/path", handler) or mux.HandleFunc("/path", handler)
+            // net/http
             const pattern = /(?:HandleFunc|Handle)\s*\(\s*["']([^"']+)["']/g;
             let match;
             while ((match = pattern.exec(content)) !== null) {
-                routes.push({
-                    method: "ALL",
-                    path: match[1],
-                    file: rel,
-                    tags: detectTags(content),
-                    framework: fw,
-                });
+                // Go 1.22+: "GET /path" patterns
+                const pathStr = match[1];
+                let method = "ALL";
+                let path = pathStr;
+                const methodMatch = pathStr.match(/^(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)\s+(\/.*)/);
+                if (methodMatch) {
+                    method = methodMatch[1];
+                    path = methodMatch[2];
+                }
+                routes.push({ method, path, file: rel, tags, framework: fw });
             }
         }
     }