agent-security-scanner-mcp 1.5.0 → 2.0.1

package/analyzer.py

@@ -1,85 +1,87 @@
+"""
+Security Analyzer - AST-Based with Regex Fallback
+
+Uses tree-sitter AST analysis when available, falls back to regex
+pattern matching when tree-sitter is not installed. This ensures
+the analyzer works out-of-the-box with `npx` (regex mode) and
+provides enhanced detection when dependencies are installed.
+"""
+
 import sys
 import json
-import re
 import os
+import re
+from typing import List, Dict, Any
 
 # Add the directory containing this script to the path
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-from rules import get_rules, get_rules_for_language, get_rule_stats
+# Try to import AST engine
+try:
+    from ast_parser import ASTParser, HAS_TREE_SITTER
+    from generic_ast import convert_tree
+    from pattern_matcher import RuleEngine
+    from regex_fallback import apply_regex_fallback
+    HAS_AST_ENGINE = HAS_TREE_SITTER
+except ImportError:
+    HAS_AST_ENGINE = False
+
+# Try to import Semgrep loader and taint analyzer
+try:
+    from semgrep_loader import load_rules, get_loader
+    HAS_SEMGREP_LOADER = True
+except ImportError:
+    HAS_SEMGREP_LOADER = False
+
+try:
+    from taint_analyzer import TaintAnalyzer
+    HAS_TAINT_ANALYZER = True
+except ImportError:
+    HAS_TAINT_ANALYZER = False
+
+# Import the original regex-based rules (always available)
+from rules import get_rules_for_language
 
 # File extension to language mapping
 EXTENSION_MAP = {
-    '.py': 'python',
-    '.js': 'javascript',
-    '.ts': 'typescript',
-    '.tsx': 'typescript',
-    '.jsx': 'javascript',
-    '.java': 'java',
-    '.go': 'go',
-    '.rb': 'ruby',
-    '.php': 'php',
-    '.cs': 'csharp',
-    '.rs': 'rust',
-    '.c': 'c',
-    '.cpp': 'cpp',
-    '.h': 'c',
-    '.hpp': 'cpp',
-    '.sql': 'sql',
-    '.dockerfile': 'dockerfile',
-    '.yaml': 'yaml',
-    '.yml': 'yaml',
-    '.json': 'json',
-    '.tf': 'terraform',
-    '.hcl': 'terraform',
-    # Prompt/text file extensions for prompt injection scanning
-    '.txt': 'generic',
-    '.md': 'generic',
-    '.prompt': 'generic',
-    '.jinja': 'generic',
-    '.jinja2': 'generic',
-    '.j2': 'generic',
+    '.py': 'python', '.js': 'javascript', '.ts': 'typescript',
+    '.tsx': 'typescript', '.jsx': 'javascript', '.java': 'java',
+    '.go': 'go', '.rb': 'ruby', '.php': 'php', '.cs': 'csharp',
+    '.rs': 'rust', '.c': 'c', '.cpp': 'cpp', '.h': 'c', '.hpp': 'cpp',
+    '.sql': 'sql', '.dockerfile': 'dockerfile',
+    '.yaml': 'yaml', '.yml': 'yaml', '.json': 'json',
+    '.tf': 'terraform', '.hcl': 'terraform',
+    '.txt': 'generic', '.md': 'generic', '.prompt': 'generic',
+    '.jinja': 'generic', '.jinja2': 'generic', '.j2': 'generic',
 }
 
+
 def detect_language(file_path):
-    """Detect the programming language from file extension or name"""
+    """Detect the programming language from file extension or name."""
     basename = os.path.basename(file_path).lower()
-    
     if basename == 'dockerfile' or basename.startswith('dockerfile.'):
         return 'dockerfile'
-    
     _, ext = os.path.splitext(file_path.lower())
     return EXTENSION_MAP.get(ext, 'generic')
 
-def analyze_file(file_path):
-    """Analyze a single file for security vulnerabilities"""
+
+def analyze_file_regex(file_path):
+    """Original regex-based analysis (fallback when tree-sitter unavailable)."""
     issues = []
-    
     try:
         language = detect_language(file_path)
         rules = get_rules_for_language(language)
-        
         with open(file_path, 'r', encoding='utf-8') as f:
             lines = f.readlines()
-            content = ''.join(lines)
-        
-        for line_index, line in enumerate(lines):
-            original_line = line
-            line = line.strip()
-            if not line:
-                continue
-            
-            # Skip comment-only lines (basic detection)
-            if line.startswith('#') or line.startswith('//') or line.startswith('*'):
+
+        for line_index, original_line in enumerate(lines):
+            line = original_line.strip()
+            if not line or line.startswith('#') or line.startswith('//') or line.startswith('*'):
                 continue
-                
             for rule_id, rule in rules.items():
                 for pattern in rule['patterns']:
                     try:
-                        # Use IGNORECASE for better detection (API_KEY vs api_key)
-                        matches = re.finditer(pattern, line, re.IGNORECASE)
-                        for match in matches:
-                            # Calculate column based on original line (preserve indentation)
+                        for match in re.finditer(pattern, line, re.IGNORECASE):
                             col_offset = len(original_line) - len(original_line.lstrip())
                             issues.append({
                                 'ruleId': rule['id'],
@@ -91,36 +93,110 @@ def analyze_file(file_path):
                                 'metadata': rule.get('metadata', {})
                             })
                     except re.error:
-                        # Skip invalid regex patterns
                         continue
-                        
     except Exception as e:
         return {'error': str(e)}
-    
-    # Deduplicate issues (same rule, same line)
+
     seen = set()
-    unique_issues = []
+    unique = []
     for issue in issues:
         key = (issue['ruleId'], issue['line'], issue['column'])
         if key not in seen:
             seen.add(key)
-            unique_issues.append(issue)
-        
-    return unique_issues
+            unique.append(issue)
+    return unique
+
+
+def analyze_file_ast(file_path):
+    """AST-based analysis using tree-sitter."""
+    try:
+        parser = ASTParser()
+        engine = RuleEngine()
+
+        # Load rules
+        rules = []
+        taint_rules = []
+        if HAS_SEMGREP_LOADER:
+            supported = ['python', 'javascript', 'typescript', 'java', 'go',
+                         'ruby', 'php', 'c', 'rust', 'csharp', 'generic']
+            rules = load_rules(supported)
+            loader = get_loader()
+            taint_rules = loader.get_taint_rules()
+
+        parse_result = parser.parse_file(file_path)
+        if not parse_result.success:
+            # Fall back to regex if AST parse fails
+            return analyze_file_regex(file_path)
+
+        ast = convert_tree(parse_result.tree, parse_result.language, parse_result.source_bytes)
+
+        applicable_rules = [
+            r for r in rules
+            if parse_result.language in r.languages or 'generic' in r.languages
+        ]
+
+        findings = engine.apply_rules(applicable_rules, ast)
+
+        # Taint analysis
+        if HAS_TAINT_ANALYZER and taint_rules:
+            taint = TaintAnalyzer()
+            applicable_taint = [
+                r for r in taint_rules
+                if parse_result.language in r.languages or 'generic' in r.languages
+            ]
+            findings.extend(taint.analyze(ast, applicable_taint))
+
+        issues = []
+        for f in findings:
+            length = f.end_column - f.column if f.line == f.end_line else len(f.text)
+            issues.append({
+                'ruleId': f.rule_id,
+                'message': f"[{f.rule_name}] {f.message}",
+                'line': f.line - 1,  # Convert to 0-indexed for compatibility
+                'column': f.column,
+                'length': length,
+                'severity': f.severity,
+                'metadata': f.metadata,
+            })
+
+        # Regex fallback for coverage gaps
+        source = parse_result.source_bytes.decode('utf-8', errors='replace')
+        issues.extend(apply_regex_fallback(source, parse_result.language, file_path))
+
+        seen = set()
+        unique = []
+        for issue in issues:
+            key = (issue['ruleId'], issue['line'], issue['column'])
+            if key not in seen:
+                seen.add(key)
+                unique.append(issue)
+        return unique
+
+    except Exception:
+        # Fall back to regex on any AST engine error
+        return analyze_file_regex(file_path)
+
+
+def analyze_file(file_path):
+    """Analyze a file — uses AST engine if available, regex otherwise."""
+    if HAS_AST_ENGINE:
+        return analyze_file_ast(file_path)
+    return analyze_file_regex(file_path)
+
 
 def main():
     if len(sys.argv) < 2:
         print(json.dumps({'error': 'No file path provided'}))
         sys.exit(1)
-        
+
     file_path = sys.argv[1]
-    
     if not os.path.exists(file_path):
         print(json.dumps({'error': f'File not found: {file_path}'}))
         sys.exit(1)
-        
+
     results = analyze_file(file_path)
     print(json.dumps(results))
 
+
 if __name__ == '__main__':
     main()

package/ast_parser.py

@@ -0,0 +1,296 @@
+"""
+AST Parser Module - tree-sitter Integration
+
+This module provides multi-language AST parsing using tree-sitter.
+Supports: Python, JavaScript, TypeScript, Java, Go, Ruby, PHP, C, C++, Rust, C#
+"""
+
+import os
+from typing import Optional, Dict, Any, List
+from dataclasses import dataclass
+
+# tree-sitter imports
+try:
+    import tree_sitter_python as tspython
+    import tree_sitter_javascript as tsjavascript
+    import tree_sitter_java as tsjava
+    import tree_sitter_go as tsgo
+    import tree_sitter_ruby as tsruby
+    import tree_sitter_php as tsphp
+    import tree_sitter_c as tsc
+    import tree_sitter_cpp as tscpp
+    import tree_sitter_rust as tsrust
+    import tree_sitter_c_sharp as tscsharp
+    import tree_sitter_typescript as tstypescript
+    from tree_sitter import Language, Parser
+    HAS_TREE_SITTER = True
+except ImportError:
+    HAS_TREE_SITTER = False
+    # Define stub types for type hints when tree-sitter not installed
+    Parser = None
+    Language = None
+
+
+# Language registry - maps file extensions to tree-sitter languages
+LANGUAGE_REGISTRY: Dict[str, Any] = {}
+
+if HAS_TREE_SITTER:
+    LANGUAGE_REGISTRY = {
+        'python': Language(tspython.language()),
+        'javascript': Language(tsjavascript.language()),
+        'typescript': Language(tstypescript.language_typescript()),
+        'tsx': Language(tstypescript.language_tsx()),
+        'java': Language(tsjava.language()),
+        'go': Language(tsgo.language()),
+        'ruby': Language(tsruby.language()),
+        'php': Language(tsphp.language_php()),
+        'c': Language(tsc.language()),
+        'cpp': Language(tscpp.language()),
+        'rust': Language(tsrust.language()),
+        'csharp': Language(tscsharp.language()),
+    }
+
+
+# File extension to language mapping
+EXTENSION_MAP = {
+    '.py': 'python',
+    '.js': 'javascript',
+    '.jsx': 'javascript',
+    '.ts': 'typescript',
+    '.tsx': 'tsx',
+    '.java': 'java',
+    '.go': 'go',
+    '.rb': 'ruby',
+    '.php': 'php',
+    '.c': 'c',
+    '.h': 'c',
+    '.cpp': 'cpp',
+    '.cc': 'cpp',
+    '.cxx': 'cpp',
+    '.hpp': 'cpp',
+    '.rs': 'rust',
+    '.cs': 'csharp',
+    # Additions for fallback support
+    '.yaml': 'kubernetes', 
+    '.yml': 'kubernetes',
+    '.tf': 'terraform',
+    '.txt': 'generic',
+    '.md': 'generic',
+    '.json': 'json',
+}
+
+
+@dataclass
+class ParseResult:
+    """Result of parsing a source file"""
+    tree: Any  # tree_sitter.Tree or MockTree
+    language: str
+    source_bytes: bytes
+    success: bool
+    error: Optional[str] = None
+
+
+class MockNode:
+    """Mock tree-sitter node for fallback parsing"""
+    def __init__(self, type_name: str, source_bytes: bytes):
+        self.type = type_name
+        self.start_byte = 0
+        self.end_byte = len(source_bytes)
+        self.start_point = (0, 0)
+        self.end_point = (source_bytes.count(b'\n'), 0)
+        self.child_count = 0
+        self.children = []
+
+
+class MockTree:
+    """Mock tree-sitter tree for fallback parsing"""
+    def __init__(self, source_bytes: bytes, language: str):
+        self.root_node = MockNode("source_file", source_bytes)
+        self.language = language
+
+
+class ASTParser:
+    """
+    Multi-language AST parser using tree-sitter.
+    
+    Example usage:
+        parser = ASTParser()
+        result = parser.parse_file("example.py")
+        if result.success:
+            root = result.tree.root_node
+            print(root.sexp())
+    """
+    
+    def __init__(self):
+        if not HAS_TREE_SITTER:
+            raise ImportError(
+                "tree-sitter and language bindings are required. "
+                "Install with: pip install -r requirements.txt"
+            )
+        self._parsers: Dict[str, Parser] = {}
+    
+    def _get_parser(self, language: str) -> Optional[Parser]:
+        """Get or create a parser for the specified language"""
+        if language not in LANGUAGE_REGISTRY:
+            return None
+        
+        if language not in self._parsers:
+            parser = Parser()
+            parser.language = LANGUAGE_REGISTRY[language]
+            self._parsers[language] = parser
+        
+        return self._parsers[language]
+    
+    def detect_language(self, file_path: str) -> Optional[str]:
+        """Detect programming language from file extension"""
+        _, ext = os.path.splitext(file_path.lower())
+        return EXTENSION_MAP.get(ext)
+    
+    def parse_file(self, file_path: str) -> ParseResult:
+        """Parse a source file and return the AST"""
+        language = self.detect_language(file_path)
+        
+        if not language:
+            # Fallback for unknown extensions
+            language = 'generic'
+        
+        parser = self._get_parser(language)
+        if not parser:
+            # Fallback to generic parsing for unsupported languages
+            try:
+                with open(file_path, 'rb') as f:
+                    source_bytes = f.read()
+                return ParseResult(
+                    tree=MockTree(source_bytes, language),
+                    language=language,
+                    source_bytes=source_bytes,
+                    success=True
+                )
+            except Exception as e:
+                return ParseResult(
+                    tree=None,
+                    language=language,
+                    source_bytes=b'',
+                    success=False,
+                    error=str(e)
+                )
+        
+        try:
+            with open(file_path, 'rb') as f:
+                source_bytes = f.read()
+            
+            tree = parser.parse(source_bytes)
+            
+            return ParseResult(
+                tree=tree,
+                language=language,
+                source_bytes=source_bytes,
+                success=True
+            )
+        except Exception as e:
+            return ParseResult(
+                tree=None,
+                language=language,
+                source_bytes=b'',
+                success=False,
+                error=str(e)
+            )
+    
+    def parse_string(self, source: str, language: str) -> ParseResult:
+        """Parse source code string and return the AST"""
+        parser = self._get_parser(language)
+        if not parser:
+            return ParseResult(
+                tree=None,
+                language=language,
+                source_bytes=b'',
+                success=False,
+                error=f"No parser available for language: {language}"
+            )
+        
+        try:
+            source_bytes = source.encode('utf-8')
+            tree = parser.parse(source_bytes)
+            
+            return ParseResult(
+                tree=tree,
+                language=language,
+                source_bytes=source_bytes,
+                success=True
+            )
+        except Exception as e:
+            return ParseResult(
+                tree=None,
+                language=language,
+                source_bytes=b'',
+                success=False,
+                error=str(e)
+            )
+    
+    def get_supported_languages(self) -> List[str]:
+        """Return list of supported languages"""
+        return list(LANGUAGE_REGISTRY.keys())
+
+
+def walk_tree(node, callback, depth=0):
+    """
+    Walk the AST tree and call callback for each node.
+    
+    Args:
+        node: tree_sitter.Node to walk
+        callback: function(node, depth) -> bool, return False to stop descent
+        depth: current depth in tree
+    """
+    if callback(node, depth) is False:
+        return
+    
+    for child in node.children:
+        walk_tree(child, callback, depth + 1)
+
+
+def get_node_text(node, source_bytes: bytes) -> str:
+    """Extract the text of a node from source bytes"""
+    return source_bytes[node.start_byte:node.end_byte].decode('utf-8')
+
+
+def find_nodes_by_type(root_node, node_type: str) -> List[Any]:
+    """Find all nodes of a specific type in the tree"""
+    results = []
+    
+    def collect(node, depth):
+        if node.type == node_type:
+            results.append(node)
+        return True
+    
+    walk_tree(root_node, collect)
+    return results
+
+
+# Convenience function for quick parsing
+def parse(file_path: str) -> ParseResult:
+    """Quick parse a file - creates a new parser each time"""
+    parser = ASTParser()
+    return parser.parse_file(file_path)
+
+
+if __name__ == '__main__':
+    import sys
+    
+    if len(sys.argv) < 2:
+        print("Usage: python ast_parser.py <file_path>")
+        sys.exit(1)
+    
+    result = parse(sys.argv[1])
+    
+    if result.success:
+        print(f"Language: {result.language}")
+        print(f"Root node type: {result.tree.root_node.type}")
+        print(f"Child count: {result.tree.root_node.child_count}")
+        
+        # Print first few child nodes as demo
+        print("\nTop-level nodes:")
+        for i, child in enumerate(result.tree.root_node.children[:10]):
+            text_preview = get_node_text(child, result.source_bytes)[:50].replace('\n', '\\n')
+            print(f"  [{i}] {child.type}: {text_preview}...")
+    else:
+        print(f"Error: {result.error}")