@dtt_siye/atool 1.1.0

package/lib/pre-scan.sh

@@ -0,0 +1,1045 @@
+#!/usr/bin/env bash
+# aTool - lib/pre-scan.sh
+# Fast structural metadata extraction using grep/awk/jq — no AI required
+#
+# Extracts per-file: imports, exports, classes, functions, decorators,
+# API endpoints, call references, and data models. Output feeds directly
+# into project-analyze Phase 1 (inventory) and Phase 2 (deep analysis)
+# to eliminate redundant AI file reads.
+#
+# Usage: source lib/pre-scan.sh
+#   pre_scan_project <PROJECT_ROOT> <OUTPUT_DIR>
+
+set -euo pipefail
+
+# shellcheck source=lib/common.sh
+# shellcheck disable=SC1091
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+# === Configuration ===
+
+PRESCAN_EXCLUDE_DIRS=(
+    "node_modules" ".git" "dist" "build" "target" "vendor"
+    "__pycache__" ".next" ".nuxt" ".output" ".atool-docs"
+    "coverage" ".cache" ".turbo" ".gradle" ".mvn"
+    "migrations" "alembic" "htmlcov" ".idea" ".vscode"
+    "gen" "generated" "proto" ".venv" "venv" "env"
+)
+
+PRESCAN_EXCLUDE_PATTERNS=(
+    "*.generated.*" "*.pb.go" "*.min.js" "*.min.css"
+    "*.bundle.js" "*.d.ts" "__pycache__"
+)
+
+# Language detection is done via detect_file_language() below (case-based, bash 3.2 compatible)
+
+# === Language Detection ===
+
+# Detect language from file extension
+# Args: FILE_PATH
+# Outputs: language name or "unknown"
+detect_file_language() {
+    local file="$1"
+    local ext=".${file##*.}"
+    case "$ext" in
+        .py|.pyw)       echo "python" ;;
+        .java)          echo "java" ;;
+        .kt|.kts)       echo "kotlin" ;;
+        .ts|.tsx)       echo "typescript" ;;
+        .js|.jsx)       echo "javascript" ;;
+        .go)            echo "go" ;;
+        .rs)            echo "rust" ;;
+        .swift)         echo "swift" ;;
+        .dart)          echo "dart" ;;
+        .vue)           echo "vue" ;;
+        .svelte)        echo "svelte" ;;
+        .ets)           echo "harmony" ;;
+        .c|.h)          echo "c" ;;
+        .cpp|.hpp)      echo "cpp" ;;
+        *)              echo "unknown" ;;
+    esac
+}
+
+# === Exclude Helper ===
+
+# Build find exclude arguments
+# Outputs: space-separated -not -path args
+_prescan_exclude_args() {
+    local args=""
+    for d in "${PRESCAN_EXCLUDE_DIRS[@]}"; do
+        args+="-not -path */${d}/* "
+    done
+    printf '%s' "$args"
+}
+
+# Check if a file should be excluded
+# Args: FILE_PATH
+# Returns: 0 (include), 1 (exclude)
+_should_scan_file() {
+    local file="$1"
+    for d in "${PRESCAN_EXCLUDE_DIRS[@]}"; do
+        [[ "$file" == *"/${d}/"* ]] && return 1
+    done
+    for pat in "${PRESCAN_EXCLUDE_PATTERNS[@]}"; do
+        # Check filename against glob pattern
+        case "$(basename "$file")" in
+            $pat) return 1 ;;
+        esac
+    done
+    return 0
+}
+
+# === Python Extractor ===
+
+# Extract metadata from a Python file using grep+awk
+# Args: FILE_PATH, RELATIVE_PATH
+# Outputs: JSON object to stdout
+_extract_python() {
+    local file="$1"
+    local rel_path="$2"
+    local hash line_count
+
+    hash=$(file_checksum "$file")
+    line_count=$(wc -l < "$file" | tr -d ' ')
+
+    # --- Imports ---
+    local imports_json="[]"
+    imports_json=$({ grep -n -E '^(from |import )' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        if (content ~ /^from /) {
+            gsub(/^from /, "", content)
+            split(content, parts, " import ")
+            mod = parts[1]
+            syms = parts[2]
+            gsub(/ /, "", syms)
+            n = split(syms, s, ",")
+            for (i = 1; i <= n; i++) {
+                if (s[i] != "") {
+                    is_local = (mod ~ /^\./ || mod ~ /^app\./ || mod ~ /^src\./) ? "true" : "false"
+                    printf "{\"from\":\"%s\",\"import\":\"%s\",\"line\":%s,\"is_local\":%s}\n", mod, s[i], line, is_local
+                }
+            }
+        } else if (content ~ /^import /) {
+            gsub(/^import /, "", content)
+            gsub(/ /, "", content)
+            n = split(content, s, ",")
+            for (i = 1; i <= n; i++) {
+                if (s[i] != "") {
+                    printf "{\"from\":null,\"import\":\"%s\",\"line\":%s,\"is_local\":false}\n", s[i], line
+                }
+            }
+        }
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Classes ---
+    local classes_json="[]"
+    classes_json=$({ grep -n -E '^class ' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^class /, "", content)
+        gsub(/:.*$/, "", content)
+        # Split name and bases
+        if (index(content, "(") > 0) {
+            split(content, parts, "(")
+            name = parts[1]
+            bases_str = content
+            sub(/^[^(]*\(/, "", bases_str)
+            sub(/\).*$/, "", bases_str)
+            gsub(/ /, "", bases_str)
+            n = split(bases_str, b, ",")
+            bases = ""
+            for (i = 1; i <= n; i++) {
+                if (b[i] != "") bases = bases (bases ? "," : "") "\"" b[i] "\""
+            }
+        } else {
+            name = content
+            bases = ""
+        }
+        gsub(/ /, "", name)
+        printf "{\"name\":\"%s\",\"bases\":[%s],\"line\":%s}\n", name, bases, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Functions ---
+    local functions_json="[]"
+    functions_json=$({ grep -n -E '^(    )?def |^(    )?async def ' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^(    )?(async )?def /, "", content)
+        # Extract name and params
+        if (index(content, "(") > 0) {
+            split(content, parts, "(")
+            name = parts[1]
+            params = content
+            sub(/^[^(]*\(/, "", params)
+            sub(/\).*$/, "", params)
+            gsub(/ /, "", params)
+        } else {
+            name = content
+            params = ""
+        }
+        is_method = ($2 ~ /^    /) ? "true" : "false"
+        gsub(/ /, "", name)
+        printf "{\"name\":\"%s\",\"params\":\"%s\",\"line\":%s,\"is_method\":%s}\n", name, params, line, is_method
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Decorators ---
+    local decorators_json="[]"
+    decorators_json=$({ grep -n -E '^@' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^@/, "", content)
+        gsub(/ *$/, "", content)
+        # Check for API route decorators
+        is_route = (index(content,".get(")>0||index(content,".post(")>0||index(content,".put(")>0||index(content,".delete(")>0||index(content,".patch(")>0||index(content,".api_route(")>0||index(content,".route(")>0) ? "true" : "false"
+        printf "{\"name\":\"%s\",\"line\":%s,\"is_route\":%s}\n", content, line, is_route
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- API Endpoints ---
+    local endpoints_json="[]"
+    # FastAPI/Sanic/Falcon route patterns
+    endpoints_json=$({ grep -n -E '@(app|router|bp|blueprint|api)\.(get|post|put|delete|patch|api_route|route)\(' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        # Extract decorator name and path
+        gsub(/^@/, "", content)
+        method = "unknown"
+        if (index(content, ".get(") > 0) method = "GET"
+        else if (index(content, ".post(") > 0) method = "POST"
+        else if (index(content, ".put(") > 0) method = "PUT"
+        else if (index(content, ".delete(") > 0) method = "DELETE"
+        else if (index(content, ".patch(") > 0) method = "PATCH"
+        # Extract path argument
+        path = ""
+        if (match(content, /\("[^"]*"/)) {
+            path = substr(content, RSTART+2, RLENGTH-3)
+        }
+        printf "{\"method\":\"%s\",\"path\":\"%s\",\"line\":%s}\n", method, path, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Data Models (SQLAlchemy/Tortoise/Pydantic) ---
+    local models_json="[]"
+    models_json=$({ grep -n -E 'class .+\((.*Model.*|.*Base.*|.*Schema.*|.*Entity.*|.*Table.*|.*Mixin.*)\)' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^class /, "", content)
+        split(content, parts, "(")
+        name = parts[1]
+        gsub(/ /, "", name)
+        bases = content
+        sub(/^[^(]*\(/, "", bases)
+        sub(/\).*$/, "", bases)
+        model_type = "unknown"
+        if (bases ~ /Base/ || bases ~ /Model/) model_type = "orm_model"
+        else if (bases ~ /Schema/ || bases ~ /BaseModel/) model_type = "pydantic"
+        else if (bases ~ /Table/) model_type = "table"
+        printf "{\"name\":\"%s\",\"type\":\"%s\",\"line\":%s}\n", name, model_type, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Exports (module-level names, i.e. non-method functions + classes) ---
+    local exports_json="[]"
+    exports_json=$(jq -n \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        '[$classes[] | {name: .name, type: "class", line: .line}] +
+         [$functions[] | select(.is_method == false) | {name: .name, type: "function", line: .line}]' 2>/dev/null || echo '[]')
+
+    # --- Assemble file JSON ---
+    jq -n \
+        --arg path "$rel_path" \
+        --arg hash "$hash" \
+        --argjson lines "$line_count" \
+        --argjson imports "$imports_json" \
+        --argjson exports "$exports_json" \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        --argjson decorators "$decorators_json" \
+        --argjson api_endpoints "$endpoints_json" \
+        --argjson data_models "$models_json" \
+        '{
+            path: $path,
+            hash: $hash,
+            lines: $lines,
+            language: "python",
+            imports: $imports,
+            exports: $exports,
+            classes: $classes,
+            functions: $functions,
+            decorators: $decorators,
+            api_endpoints: $api_endpoints,
+            data_models: $data_models,
+            extraction_confidence: "high"
+        }'
+}
+
+# === Java/Kotlin Extractor ===
+
+_extract_java() {
+    local file="$1"
+    local rel_path="$2"
+    local hash line_count
+
+    hash=$(file_checksum "$file")
+    line_count=$(wc -l < "$file" | tr -d ' ')
+
+    # Detect if Kotlin
+    local lang="java"
+    [[ "$rel_path" == *.kt || "$rel_path" == *.kts ]] && lang="kotlin"
+
+    # --- Imports ---
+    local imports_json="[]"
+    imports_json=$({ grep -n -E '^import ' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^import /, "", content)
+        gsub(/;.*$/, "", content)
+        gsub(/ *$/, "", content)
+        is_local = (content ~ /^com\.example/) ? "false" : "false"
+        # Java imports are almost always external
+        printf "{\"from\":\"%s\",\"import\":null,\"line\":%s,\"is_local\":false}\n", content, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Classes/Interfaces/Enums ---
+    local classes_json="[]"
+    classes_json=$({ grep -n -E '(public |private |protected )?(static )?(abstract )?(final )?(class |interface |enum |@interface )' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/public |private |protected |static |abstract |final |/, "", content)
+        decl_type = "class"
+        if (content ~ /^interface /) decl_type = "interface"
+        else if (content ~ /^enum /) decl_type = "enum"
+        else if (content ~ /^@interface /) decl_type = "annotation"
+        gsub(/^(class |interface |enum |@interface )/, "", content)
+        # Extract name and possible extends/implements
+        name = content
+        gsub(/[ ({:].*$/, "", name)
+        gsub(/ /, "", name)
+        printf "{\"name\":\"%s\",\"type\":\"%s\",\"line\":%s}\n", name, decl_type, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Methods ---
+    local functions_json="[]"
+    functions_json=$({ grep -n -E '(public|private|protected).*\([^)]*\)\s*(\{|throws|@)' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        # Simplified method signature extraction
+        gsub(/public |private |protected |static |abstract |final |synchronized |/, "", content)
+        gsub(/ *$/, "", content)
+        # Extract return type and method name
+        if (match(content, /[a-zA-Z_][a-zA-Z0-9_]*\(/)) {
+            pre = substr(content, 1, RSTART - 1)
+            method_with_paren = substr(content, RSTART)
+            gsub(/\(.*/, "", method_with_paren)
+            printf "{\"name\":\"%s\",\"params\":\"\",\"line\":%s}\n", method_with_paren, line
+        }
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Annotations (Spring) ---
+    local decorators_json="[]"
+    decorators_json=$({ grep -n -E '@\w+' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/ /, "", content)
+        gsub(/^ */, "", content)
+        is_route = (content ~ /Mapping|Controller|RestController|RequestMapping/) ? "true" : "false"
+        printf "{\"name\":\"%s\",\"line\":%s,\"is_route\":%s}\n", content, line, is_route
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- API Endpoints (Spring) ---
+    local endpoints_json="[]"
+    endpoints_json=$({ grep -n -E '@(Get|Post|Put|Delete|Patch|Request)Mapping' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        method = "unknown"
+        if (index(content, "@GetMapping") > 0) method = "GET"
+        else if (index(content, "@PostMapping") > 0) method = "POST"
+        else if (index(content, "@PutMapping") > 0) method = "PUT"
+        else if (index(content, "@DeleteMapping") > 0) method = "DELETE"
+        else if (index(content, "@PatchMapping") > 0) method = "PATCH"
+        else if (index(content, "@RequestMapping") > 0) method = "ANY"
+        path = ""
+        if (match(content, /\("[^"]*"/)) {
+            path = substr(content, RSTART+2, RLENGTH-3)
+        } else if (index(content, "/") > 0) {
+            if (match(content, /"[^"]*"/)) {
+                path = substr(content, RSTART+1, RLENGTH-2)
+            }
+        }
+        printf "{\"method\":\"%s\",\"path\":\"%s\",\"line\":%s}\n", method, path, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Data Models (JPA/Entity) ---
+    local models_json="[]"
+    models_json=$({ grep -n -E '@(Entity|Table)' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1
+        printf "{\"type\":\"jpa_entity\",\"line\":%s}\n", line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Exports ---
+    local exports_json
+    exports_json=$(jq -n \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        '[$classes[] | {name: .name, type: .type, line: .line}] +
+         [$functions[] | {name: .name, type: "method", line: .line}]' 2>/dev/null || echo '[]')
+
+    # --- Assemble ---
+    jq -n \
+        --arg path "$rel_path" \
+        --arg lang "$lang" \
+        --arg hash "$hash" \
+        --argjson lines "$line_count" \
+        --argjson imports "$imports_json" \
+        --argjson exports "$exports_json" \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        --argjson decorators "$decorators_json" \
+        --argjson api_endpoints "$endpoints_json" \
+        --argjson data_models "$models_json" \
+        '{
+            path: $path,
+            hash: $hash,
+            lines: $lines,
+            language: $lang,
+            imports: $imports,
+            exports: $exports,
+            classes: $classes,
+            functions: $functions,
+            decorators: $decorators,
+            api_endpoints: $api_endpoints,
+            data_models: $data_models,
+            extraction_confidence: "high"
+        }'
+}
+
+# === TypeScript/JavaScript Extractor ===
+
+_extract_typescript() {
+    local file="$1"
+    local rel_path="$2"
+    local hash line_count
+
+    hash=$(file_checksum "$file")
+    line_count=$(wc -l < "$file" | tr -d ' ')
+
+    local lang="typescript"
+    [[ "$rel_path" == *.js || "$rel_path" == *.jsx ]] && lang="javascript"
+
+    # --- Imports ---
+    local imports_json="[]"
+    imports_json=$({ grep -n -E '^(import |const .* = require)' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/\047/, "\"", content)
+        if (content ~ /^import /) {
+            # ES module import
+            source = ""
+            if (match(content, /from "[^"]*"/)) {
+                source = substr(content, RSTART+6, RLENGTH-7)
+            } else if (match(content, /"[^"]*"/)) {
+                source = substr(content, RSTART+1, RLENGTH-2)
+            }
+            is_local = (source ~ /^\./ || source ~ /^@/) ? "true" : "false"
+            printf "{\"from\":\"%s\",\"import\":null,\"line\":%s,\"is_local\":%s}\n", source, line, is_local
+        } else if (index(content, "require(") > 0) {
+            source = ""
+            if (match(content, /"[^"]*"/)) {
+                source = substr(content, RSTART+1, RLENGTH-2)
+            }
+            is_local = (source ~ /^\./ || source ~ /^@/) ? "true" : "false"
+            printf "{\"from\":\"%s\",\"import\":null,\"line\":%s,\"is_local\":%s}\n", source, line, is_local
+        }
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Classes/Interfaces/Types/Enums ---
+    local classes_json="[]"
+    classes_json=$({ grep -n -E '^(export )?(default )?(abstract )?(class |interface |type |enum )' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        is_export = (content ~ /^export /) ? "true" : "false"
+        gsub(/^export (default )?/, "", content)
+        gsub(/^abstract /, "", content)
+        decl_type = "class"
+        if (content ~ /^interface /) decl_type = "interface"
+        else if (content ~ /^type /) decl_type = "type"
+        else if (content ~ /^enum /) decl_type = "enum"
+        gsub(/^(class |interface |type |enum )/, "", content)
+        name = content
+        gsub(/[ <({:].*$/, "", name)
+        gsub(/ /, "", name)
+        printf "{\"name\":\"%s\",\"type\":\"%s\",\"line\":%s,\"exported\":%s}\n", name, decl_type, line, is_export
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Functions/Arrow Functions ---
+    local functions_json="[]"
+    functions_json=$({ grep -n -E '^(export )?(default )?(async )?(function |const [a-zA-Z]+ = (\(.*\) =>|\(.*\):))' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^export (default )?/, "", content)
+        gsub(/^async /, "", content)
+        name = ""
+        if (content ~ /^function /) {
+            gsub(/^function /, "", content)
+            name = content
+            gsub(/[( ].*$/, "", name)
+        } else if (content ~ /^const /) {
+            gsub(/^const /, "", content)
+            name = content
+            gsub(/ *=.*$/, "", name)
+        }
+        gsub(/ /, "", name)
+        if (name != "") {
+            printf "{\"name\":\"%s\",\"line\":%s}\n", name, line
+        }
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Decorators ---
+    local decorators_json="[]"
+    decorators_json=$({ grep -n -E '^\s*@\w+' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^\s*/, "", content)
+        gsub(/^@/, "", content)
+        gsub(/ *$/, "", content)
+        printf "{\"name\":\"%s\",\"line\":%s}\n", content, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- API Endpoints (Express/Fastify/Nuxt/Next) ---
+    local endpoints_json="[]"
+    endpoints_json=$({ grep -n -E '(app|router|server|fastify)\.(get|post|put|delete|patch)\(' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        method = "unknown"
+        if (index(content, ".get(") > 0) method = "GET"
+        else if (index(content, ".post(") > 0) method = "POST"
+        else if (index(content, ".put(") > 0) method = "PUT"
+        else if (index(content, ".delete(") > 0) method = "DELETE"
+        else if (index(content, ".patch(") > 0) method = "PATCH"
+        path = ""
+        if (match(content, /"[^"]*"/)) {
+            path = substr(content, RSTART+1, RLENGTH-2)
+        }
+        printf "{\"method\":\"%s\",\"path\":\"%s\",\"line\":%s}\n", method, path, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Data Models ---
+    local models_json="[]"
+    models_json=$({ grep -n -E '(interface|type|class).*(Props|State|Model|Schema|Entity|DTO|Dto|Response|Request|Input|Output)' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^(export )?(default )?(abstract )?/, "", content)
+        name = content
+        gsub(/^[a-z]+ /, "", name)
+        gsub(/[ <({:].*$/, "", name)
+        printf "{\"name\":\"%s\",\"type\":\"typescript_interface\",\"line\":%s}\n", name, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Exports ---
+    local exports_json
+    exports_json=$(jq -n \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        '[$classes[] | select(.exported == true) | {name: .name, type: .type, line: .line}] +
+         [$functions[] | {name: .name, type: "function", line: .line}]' 2>/dev/null || echo '[]')
+
+    # --- Assemble ---
+    jq -n \
+        --arg path "$rel_path" \
+        --arg lang "$lang" \
+        --arg hash "$hash" \
+        --argjson lines "$line_count" \
+        --argjson imports "$imports_json" \
+        --argjson exports "$exports_json" \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        --argjson decorators "$decorators_json" \
+        --argjson api_endpoints "$endpoints_json" \
+        --argjson data_models "$models_json" \
+        '{
+            path: $path,
+            hash: $hash,
+            lines: $lines,
+            language: $lang,
+            imports: $imports,
+            exports: $exports,
+            classes: $classes,
+            functions: $functions,
+            decorators: $decorators,
+            api_endpoints: $api_endpoints,
+            data_models: $data_models,
+            extraction_confidence: "high"
+        }'
+}
+
+# === Go Extractor ===
+
+_extract_go() {
+    local file="$1"
+    local rel_path="$2"
+    local hash line_count
+
+    hash=$(file_checksum "$file")
+    line_count=$(wc -l < "$file" | tr -d ' ')
+
+    # --- Imports ---
+    local imports_json="[]"
+    imports_json=$({ grep -n -E '^[[:space:]]+"' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^[[:space:]]*"/, "", content)
+        gsub(/".*$/, "", content)
+        if (content != "") {
+            printf "{\"from\":\"%s\",\"import\":null,\"line\":%s,\"is_local\":false}\n", content, line
+        }
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Structs and Interfaces ---
+    local classes_json="[]"
+    classes_json=$({ grep -n -E '^type \w+ (struct|interface)' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^type /, "", content)
+        if (content ~ / struct/) {
+            gsub(/ struct.*$/, "", content)
+            printf "{\"name\":\"%s\",\"type\":\"struct\",\"line\":%s}\n", content, line
+        } else if (content ~ / interface/) {
+            gsub(/ interface.*$/, "", content)
+            printf "{\"name\":\"%s\",\"type\":\"interface\",\"line\":%s}\n", content, line
+        }
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Functions ---
+    local functions_json="[]"
+    functions_json=$({ grep -n -E '^func ' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^func /, "", content)
+        # Check for method receiver
+        if (content ~ /^\(/) {
+            # Method: func (r *Receiver) Name(params)
+            sub(/^\([^)]*\)\s*/, "", content)
+        }
+        name = content
+        gsub(/[( ].*$/, "", name)
+        printf "{\"name\":\"%s\",\"line\":%s}\n", name, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- API Endpoints (net/http patterns) ---
+    local endpoints_json="[]"
+    endpoints_json=$({ grep -n -E '(http\.(HandleFunc|Handle)|\.GET\(|\.POST\(|\.PUT\(|\.DELETE\(|\.Group\()' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        method = "unknown"
+        if (index(content, ".GET(") > 0) method = "GET"
+        else if (index(content, ".POST(") > 0) method = "POST"
+        else if (index(content, ".PUT(") > 0) method = "PUT"
+        else if (index(content, ".DELETE(") > 0) method = "DELETE"
+        else if (index(content, "HandleFunc") > 0) method = "ANY"
+        path = ""
+        if (match(content, /"[^"]*"/)) {
+            path = substr(content, RSTART+1, RLENGTH-2)
+        }
+        printf "{\"method\":\"%s\",\"path\":\"%s\",\"line\":%s}\n", method, path, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Exports (capitalized names) ---
+    local exports_json
+    exports_json=$(jq -n \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        '[$classes[] | {name: .name, type: .type, line: .line}] +
+         [$functions[] | {name: .name, type: "function", line: .line}]' 2>/dev/null || echo '[]')
+
+    # --- Assemble ---
+    jq -n \
+        --arg path "$rel_path" \
+        --arg hash "$hash" \
+        --argjson lines "$line_count" \
+        --argjson imports "$imports_json" \
+        --argjson exports "$exports_json" \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        --argjson api_endpoints "$endpoints_json" \
+        '{
+            path: $path,
+            hash: $hash,
+            lines: $lines,
+            language: "go",
+            imports: $imports,
+            exports: $exports,
+            classes: $classes,
+            functions: $functions,
+            decorators: [],
+            api_endpoints: $api_endpoints,
+            data_models: [],
+            extraction_confidence: "high"
+        }'
+}
+
+# === Rust Extractor ===
+
+_extract_rust() {
+    local file="$1"
+    local rel_path="$2"
+    local hash line_count
+
+    hash=$(file_checksum "$file")
+    line_count=$(wc -l < "$file" | tr -d ' ')
+
+    # --- Imports ---
+    local imports_json="[]"
+    imports_json=$({ grep -n -E '^use ' "$file" 2>/dev/null || true; } | awk '
+    {
+        colon_pos = index($0, ":")
+        line = substr($0, 1, colon_pos - 1)
+        content = substr($0, colon_pos + 1)
+        gsub(/^use /, "", content)
+        gsub(/;.*$/, "", content)
+        gsub(/ *$/, "", content)
+        printf "{\"from\":\"%s\",\"import\":null,\"line\":%s,\"is_local\":false}\n", content, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Structs/Enums/Traits ---
+    local classes_json="[]"
+    classes_json=$({ grep -n -E '^(pub )?(struct |enum |trait )' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^pub /, "", content)
+        decl_type = "struct"
+        if (content ~ /^enum /) decl_type = "enum"
+        else if (content ~ /^trait /) decl_type = "trait"
+        gsub(/^(struct |enum |trait )/, "", content)
+        name = content
+        gsub(/[ <({:].*$/, "", name)
+        printf "{\"name\":\"%s\",\"type\":\"%s\",\"line\":%s}\n", name, decl_type, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Functions ---
+    local functions_json="[]"
+    functions_json=$({ grep -n -E '^(pub )?(async )?fn ' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^(pub )?(async )?fn /, "", content)
+        name = content
+        gsub(/[( <:].*$/, "", name)
+        printf "{\"name\":\"%s\",\"line\":%s}\n", name, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Exports ---
+    local exports_json
+    exports_json=$(jq -n \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        '[$classes[] | {name: .name, type: .type, line: .line}] +
+         [$functions[] | {name: .name, type: "function", line: .line}]' 2>/dev/null || echo '[]')
+
+    # --- Assemble ---
+    jq -n \
+        --arg path "$rel_path" \
+        --arg hash "$hash" \
+        --argjson lines "$line_count" \
+        --argjson imports "$imports_json" \
+        --argjson exports "$exports_json" \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        '{
+            path: $path,
+            hash: $hash,
+            lines: $lines,
+            language: "rust",
+            imports: $imports,
+            exports: $exports,
+            classes: $classes,
+            functions: $functions,
+            decorators: [],
+            api_endpoints: [],
+            data_models: [],
+            extraction_confidence: "high"
+        }'
+}
+
+# === Generic Extractor (fallback) ===
+
+_extract_generic() {
+    local file="$1"
+    local rel_path="$2"
+    local hash line_count
+
+    hash=$(file_checksum "$file")
+    line_count=$(wc -l < "$file" | tr -d ' ')
+
+    # --- Generic import patterns ---
+    local imports_json="[]"
+    imports_json=$({ grep -n -E '^(import |from |use |require\(|include |#include )' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/ /, "", content)
+        printf "{\"from\":null,\"import\":\"%s\",\"line\":%s,\"is_local\":false}\n", content, line
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Generic function patterns ---
+    local functions_json="[]"
+    functions_json=$({ grep -n -E '^(    )?(function |def |func |fn |sub |proc )' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^(    )?(function |def |func |fn |sub |proc )/, "", content)
+        name = content
+        gsub(/[( {:].*$/, "", name)
+        gsub(/ /, "", name)
+        if (name != "") {
+            printf "{\"name\":\"%s\",\"line\":%s}\n", name, line
+        }
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Generic class patterns ---
+    local classes_json="[]"
+    classes_json=$({ grep -n -E '^(export )?(class |struct |interface |type |enum )' "$file" 2>/dev/null || true; } | awk -F: '
+    {
+        line = $1; content = $2
+        gsub(/^export /, "", content)
+        decl_type = "class"
+        if (content ~ /^interface /) decl_type = "interface"
+        else if (content ~ /^struct /) decl_type = "struct"
+        else if (content ~ /^type /) decl_type = "type"
+        else if (content ~ /^enum /) decl_type = "enum"
+        gsub(/^(class |struct |interface |type |enum )/, "", content)
+        name = content
+        gsub(/[ <({:].*$/, "", name)
+        if (name != "") {
+            printf "{\"name\":\"%s\",\"type\":\"%s\",\"line\":%s}\n", name, decl_type, line
+        }
+    }' | jq -s '.' 2>/dev/null || echo '[]')
+
+    # --- Assemble ---
+    jq -n \
+        --arg path "$rel_path" \
+        --arg hash "$hash" \
+        --argjson lines "$line_count" \
+        --argjson imports "$imports_json" \
+        --argjson classes "$classes_json" \
+        --argjson functions "$functions_json" \
+        '{
+            path: $path,
+            hash: $hash,
+            lines: $lines,
+            language: "unknown",
+            imports: $imports,
+            exports: [],
+            classes: $classes,
+            functions: $functions,
+            decorators: [],
+            api_endpoints: [],
+            data_models: [],
+            extraction_confidence: "low"
+        }'
+}
+
+# === File Metadata Dispatcher ===
+
+# Extract metadata from a single file
+# Args: FILE_PATH, PROJECT_ROOT
+# Outputs: JSON object to stdout
+extract_file_metadata() {
+    local file="$1"
+    local project_root="$2"
+    local rel_path="${file#${project_root}/}"
+
+    local lang
+    lang=$(detect_file_language "$file")
+
+    case "$lang" in
+        python)    _extract_python "$file" "$rel_path" ;;
+        java|kotlin) _extract_java "$file" "$rel_path" ;;
+        typescript|javascript|vue|svelte) _extract_typescript "$file" "$rel_path" ;;
+        go)        _extract_go "$file" "$rel_path" ;;
+        rust)      _extract_rust "$file" "$rel_path" ;;
+        *)         _extract_generic "$file" "$rel_path" ;;
+    esac
+}
+
+# === Module Aggregator ===
+
+# Aggregate per-file scans into a module-level JSON
+# Args: MODULE_DIR, MODULE_SLUG, PROJECT_ROOT, OUTPUT_DIR
+aggregate_module_prescan() {
+    local module_dir="$1"
+    local module_slug="$2"
+    local project_root="$3"
+    local output_dir="$4"
+
+    local tmpdir
+    tmpdir=$(mktemp -d)
+    local file_idx=0
+
+    # Collect source files in this module
+    local exclude_args=()
+    for d in "${PRESCAN_EXCLUDE_DIRS[@]}"; do
+        exclude_args+=(-not -path "*/${d}/*")
+    done
+
+    while IFS= read -r src_file; do
+        [[ -z "$src_file" ]] && continue
+        _should_scan_file "$src_file" || continue
+
+        local result
+        if result=$(extract_file_metadata "$src_file" "$project_root" 2>/dev/null); then
+            echo "$result" > "$tmpdir/file_${file_idx}.json"
+            file_idx=$((file_idx + 1))
+        fi
+    done < <(find "$module_dir" -type f "${exclude_args[@]}" 2>/dev/null | head -500)
+
+    # Merge all file JSONs into module JSON
+    if [[ $file_idx -eq 0 ]]; then
+        echo '{"files":[],"module_summary":{"total_files":0}}'
+        rm -rf "$tmpdir"
+        return 0
+    fi
+
+    local files_array
+    files_array=$(jq -s '.' "$tmpdir"/file_*.json 2>/dev/null || echo '[]')
+
+    # Generate module summary
+    local summary
+    summary=$(echo "$files_array" | jq '{
+        total_files: length,
+        total_imports: ([.[].imports | length] | add),
+        total_exports: ([.[].exports | length] | add),
+        total_classes: ([.[].classes | length] | add),
+        total_functions: ([.[].functions | length] | add),
+        total_api_endpoints: ([.[].api_endpoints | length] | add),
+        total_data_models: ([.[].data_models | length] | add),
+        local_imports: [.[].imports[] | select(.is_local == true)] | group_by(.from) | map({module: .[0].from, count: length}),
+        external_deps: [.[].imports[] | select(.is_local == false)] | group_by(.from) | map({package: .[0].from, count: length}) | sort_by(-.count) | .[0:20]
+    }' 2>/dev/null || echo '{}')
+
+    # Assemble module JSON
+    jq -n \
+        --arg module "$module_slug" \
+        --argjson files "$files_array" \
+        --argjson summary "$summary" \
+        '{
+            module: $module,
+            scan_metadata: {
+                scan_timestamp: (now | todate),
+                extraction_method: "grep+awk+jq"
+            },
+            files: $files,
+            module_summary: $summary
+        }' > "${output_dir}/${module_slug}.json"
+
+    rm -rf "$tmpdir"
+}
+
+# === Main Entry Point ===
+
+# Pre-scan an entire project
+# Args: PROJECT_ROOT, OUTPUT_DIR
+# Outputs: writes .atool-docs/pre-scan/*.json and manifest.json
+pre_scan_project() {
+    local project_root="$1"
+    local output_dir="$2"
+
+    check_jq
+
+    ensure_dir "$output_dir"
+
+    local start_time
+    start_time=$(date +%s)
+
+    # Read module list from analysis-state.json
+    local state_file="$project_root/.atool-docs/analysis-state.json"
+    if [[ ! -f "$state_file" ]]; then
+        log_error "analysis-state.json not found — run Phase 0 discovery first"
+        return 1
+    fi
+
+    # Extract module paths and slugs from state
+    # Modules can be: (1) object keyed by slug, or (2) array of objects
+    local modules_json
+    modules_json=$(jq 'if (.modules | type) == "object" then
+        [.modules | to_entries[] | {slug: .key, path: (if .value.path then .value.path elif .value.dir then .value.dir else .key end)}]
+    elif (.modules | type) == "array" then
+        [.modules[] | {slug: (if .slug then .slug elif .name then .name else .path end), path: (if .path then .path else .dir end)}]
+    else [] end' "$state_file" 2>/dev/null)
+
+    local module_count
+    module_count=$(echo "$modules_json" | jq 'length' 2>/dev/null || echo "0")
+
+    if [[ "$module_count" -eq 0 ]]; then
+        log_warn "No modules found in analysis-state.json"
+        return 0
+    fi
+
+    log_info "Pre-scanning $module_count modules..."
+
+    local modules_processed=0
+    for ((i = 0; i < module_count; i++)); do
+        local module_slug module_path
+        module_slug=$(echo "$modules_json" | jq -r ".[$i].slug" 2>/dev/null)
+        module_path=$(echo "$modules_json" | jq -r ".[$i].path" 2>/dev/null)
+
+        [[ -z "$module_slug" || "$module_slug" == "null" ]] && continue
+        [[ -z "$module_path" || "$module_path" == "null" ]] && continue
+
+        local full_module_path="$project_root/$module_path"
+
+        if [[ ! -d "$full_module_path" ]]; then
+            log_warn "Module directory not found: $full_module_path"
+            continue
+        fi
+
+        # Check for incremental update
+        local prescan_file="$output_dir/${module_slug}.json"
+        if [[ -f "$prescan_file" ]]; then
+            # Check if any source file changed since last scan
+            local needs_rescan=false
+            local stored_hash
+            while IFS= read -r src_file; do
+                [[ -z "$src_file" ]] && continue
+                local rel_path="${src_file#${project_root}/}"
+                local current_hash
+                current_hash=$(file_checksum "$src_file")
+                stored_hash=$(jq -r --arg p "$rel_path" '.files[] | select(.path == $p) | .hash // ""' "$prescan_file" 2>/dev/null)
+                if [[ "$current_hash" != "$stored_hash" ]]; then
+                    needs_rescan=true
+                    break
+                fi
+            done < <(find "$full_module_path" -type f \( -name '*.py' -o -name '*.java' -o -name '*.kt' \
+                -o -name '*.ts' -o -name '*.tsx' -o -name '*.js' -o -name '*.jsx' \
+                -o -name '*.go' -o -name '*.rs' -o -name '*.vue' -o -name '*.svelte' \
+                -o -name '*.swift' -o -name '*.dart' -o -name '*.ets' \) 2>/dev/null | head -200)
+
+            if [[ "$needs_rescan" == "false" ]]; then
+                log_info "Module $module_slug: unchanged, skipping"
+                modules_processed=$((modules_processed + 1))
+                continue
+            fi
+        fi
+
+        log_info "Scanning module: $module_slug ($module_path)"
+        aggregate_module_prescan "$full_module_path" "$module_slug" "$project_root" "$output_dir"
+        modules_processed=$((modules_processed + 1))
+    done
+
+    # Generate manifest
+    local end_time duration
+    end_time=$(date +%s)
+    duration=$((end_time - start_time))
+
+    local manifest
+    manifest=$(jq -s '{
+        version: "1.0",
+        generated_at: (now | todate),
+        total_modules: length,
+        scan_duration_seconds: null,
+        modules: [.[].module_summary // {} | {total_files, total_classes, total_functions, total_api_endpoints, total_data_models}]
+    }' "$output_dir"/*.json 2>/dev/null || echo '{}')
+
+    # Add duration to manifest
+    manifest=$(echo "$manifest" | jq --argjson dur "$duration" '.scan_duration_seconds = $dur')
+
+    echo "$manifest" > "$output_dir/manifest.json"
+
+    log_success "Pre-scan complete: $modules_processed modules in ${duration}s"
+}