@oculum/scanner 1.0.14 → 1.0.15

package/src/model/route-discovery/python.ts

@@ -1,7 +1,8 @@
 /**
- * Python Route Extractor (Flask + Django)
+ * Python Route Extractor (Flask + Django + FastAPI)
  *
- * Detects route definitions from Flask decorators and Django URL patterns.
+ * Detects route definitions from Flask decorators, Django URL patterns,
+ * and FastAPI path operation decorators.
  */
 
 import type { RouteDefinition, RouteInput, RouteFramework } from './types'
@@ -12,14 +13,20 @@ import type { RouteDefinition, RouteInput, RouteFramework } from './types'
 
 const PUBLIC_ENDPOINT_PATTERN = /^\/(health|healthz|ready|readyz|ping|status)\/?$/i
 
-const AUTH_DECORATOR_PATTERN = /login_required|auth_required|requires_auth|permission_required|jwt_required|token_required/i
+const AUTH_DECORATOR_PATTERN = /login_required|auth_required|requires_auth|permission_required|jwt_required|token_required|fresh_jwt_required|jwt_optional|fresh_login_required|api_login_required/i
 
 const RATE_LIMIT_PATTERN = /rate_limit|ratelimit|throttle|limiter/i
 
+const FASTAPI_DEPENDS_AUTH_PATTERN = /Depends\s*\(\s*(?:get_current_user|verify_token|verify_auth|auth_required|get_user|check_auth|authenticate|require_auth)\s*\)/i
+
+const FASTAPI_SECURITY_PATTERN = /Security\s*\(/i
+
+const DJANGO_AUTH_WRAP_PATTERN = /^(login_required|permission_required\s*\([^)]*\))\s*\(\s*(.+)\s*\)$/
+
 const PYTHON_INPUT_PATTERNS: Array<{ pattern: RegExp; source: RouteInput['source'] }> = [
   { pattern: /request\.json|request\.get_json|request\.data|request\.form|request\.POST|request\.body/i, source: 'body' },
   { pattern: /request\.args|request\.GET|request\.query_params/i, source: 'query' },
-  { pattern: /request\.view_args|kwargs\[|self\.kwargs/i, source: 'params' },
+  { pattern: /request\.view_args|kwargs\[|self\.kwargs|request\.path_params/i, source: 'params' },
   { pattern: /request\.headers|request\.META/i, source: 'headers' },
   { pattern: /request\.cookies|request\.COOKIES/i, source: 'cookies' },
   { pattern: /request\.files|request\.FILES/i, source: 'files' },
@@ -30,12 +37,14 @@ const PYTHON_INPUT_PATTERNS: Array<{ pattern: RegExp; source: RouteInput['source
 // ============================================================================
 
 /**
- * Extract Flask/Django routes from Python file content.
+ * Extract Flask/Django/FastAPI routes from Python file content.
  *
  * Detects:
  * - Flask: @app.route('/path', methods=['GET'])
  * - Flask: @blueprint.route(...)
  * - Django: path('api/search/', views.search_view) in urlpatterns
+ * - Django: re_path(r'^api/$', views.api_view) in urlpatterns
+ * - FastAPI: @app.get('/path'), @router.post('/path'), etc.
  */
 export function extractPythonRoutes(content: string, filePath: string): RouteDefinition[] {
   const routes: RouteDefinition[] = []
@@ -52,6 +61,12 @@ export function extractPythonRoutes(content: string, filePath: string): RouteDef
   const djangoRoutes = extractDjangoRoutes(lines, filePath, content)
   routes.push(...djangoRoutes)
 
+  // Detect FastAPI routes
+  if (/from\s+fastapi\b|import\s+fastapi\b|FastAPI\s*\(/.test(content)) {
+    const fastapiRoutes = extractFastAPIRoutes(lines, filePath, content)
+    routes.push(...fastapiRoutes)
+  }
+
   return routes
 }
 
@@ -123,13 +138,29 @@ function extractDjangoRoutes(lines: string[], filePath: string, fullContent: str
 
     // path('api/search/', views.search_view, name='search')
     // path('api/users/<int:pk>/', UserViewSet.as_view())
+    // re_path(r'^api/$', views.api_view, name='api')
+    // Capture view ref with balanced parens (up to 2 levels)
     const pathMatch = line.match(
-      /path\s*\(\s*['"]([^'"]*)['"]\s*,\s*([^,)]+)/
+      /(?:path|re_path)\s*\(\s*[r]?['"]([^'"]*)['"]\s*,\s*((?:[^,()]+|\((?:[^()]*|\([^()]*\))*\))+)/
     )
 
     if (pathMatch) {
-      const routePath = '/' + pathMatch[1].replace(/^\//, '')
-      const viewRef = pathMatch[2].trim()
+      let routePath = pathMatch[1]
+      let viewRef = pathMatch[2].trim()
+
+      // For re_path, strip regex anchors
+      routePath = routePath.replace(/^\^/, '').replace(/\$$/, '')
+
+      // Ensure leading slash
+      routePath = '/' + routePath.replace(/^\//, '')
+
+      // Detect auth wrapping: login_required(views.view) or permission_required('perm')(views.view)
+      const authMiddleware: string[] = []
+      const authWrapMatch = viewRef.match(/^(login_required|permission_required\s*\([^)]*\))\s*\(\s*(.+)\s*\)$/)
+      if (authWrapMatch) {
+        authMiddleware.push(authWrapMatch[1])
+        viewRef = authWrapMatch[2].trim()
+      }
 
       routes.push({
         filePath,
@@ -138,7 +169,7 @@ function extractDjangoRoutes(lines: string[], filePath: string, fullContent: str
         handler: viewRef.replace(/\.as_view\(\)/, ''),
         handlerLine: i + 1,
         framework: 'django',
-        authMiddleware: [],
+        authMiddleware,
         rateLimiting: false,
         isPublic: PUBLIC_ENDPOINT_PATTERN.test(routePath),
         inputs: [],
@@ -149,6 +180,102 @@ function extractDjangoRoutes(lines: string[], filePath: string, fullContent: str
   return routes
 }
 
+// ============================================================================
+// FastAPI Extraction
+// ============================================================================
+
+function extractFastAPIRoutes(lines: string[], filePath: string, content: string): RouteDefinition[] {
+  const routes: RouteDefinition[] = []
+
+  // Pre-check: must have fastapi-style decorator pattern
+  if (!/@\w+\.\s*(?:get|post|put|patch|delete|head|options|api_route)\s*\(/.test(content)) {
+    return routes
+  }
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]
+
+    // @app.get('/path'), @router.post('/path', ...), @app.api_route('/path', methods=['GET'])
+    const routeMatch = line.match(
+      /@\s*(\w+)\s*\.\s*(get|post|put|patch|delete|head|options|api_route)\s*\(\s*['"]([^'"]+)['"]/
+    )
+
+    if (!routeMatch) continue
+
+    const method = routeMatch[2]
+    const path = routeMatch[3]
+
+    // Determine HTTP methods
+    let methods: string[]
+    if (method === 'api_route') {
+      // Look for methods kwarg in the decorator
+      const methodsMatch = line.match(/methods\s*=\s*\[([^\]]*)\]/)
+      methods = methodsMatch
+        ? methodsMatch[1].replace(/['"]/g, '').split(',').map(m => m.trim().toUpperCase()).filter(Boolean)
+        : ['GET']
+    } else {
+      methods = [method.toUpperCase()]
+    }
+
+    // Check for dependencies=[Depends(verify_token)] in decorator
+    const authMiddleware: string[] = []
+    const depsMatch = line.match(/dependencies\s*=\s*\[([^\]]*)\]/)
+    if (depsMatch && FASTAPI_DEPENDS_AUTH_PATTERN.test(depsMatch[1])) {
+      const depAuthMatch = depsMatch[1].match(/Depends\s*\(\s*(\w+)\s*\)/)
+      if (depAuthMatch) {
+        authMiddleware.push(`Depends(${depAuthMatch[1]})`)
+      }
+    }
+
+    // Scan decorators above for auth/rate-limiting
+    const decorators = collectDecorators(lines, i)
+    const authDecorators = decorators.filter(d => AUTH_DECORATOR_PATTERN.test(d))
+    authMiddleware.push(...authDecorators)
+    const hasRateLimit = decorators.some(d => RATE_LIMIT_PATTERN.test(d))
+
+    // Find the function definition below
+    const handlerInfo = findFunctionBelow(lines, i)
+
+    // Extract function signature for Depends/Security in params
+    if (handlerInfo) {
+      const sig = extractFunctionSignature(lines, handlerInfo.line - 1)
+      // Check all Depends() calls in signature for auth patterns
+      if (authMiddleware.length === 0) {
+        const dependsMatches = sig.matchAll(/Depends\s*\(\s*(\w+)\s*\)/g)
+        for (const dm of dependsMatches) {
+          if (FASTAPI_DEPENDS_AUTH_PATTERN.test(dm[0])) {
+            authMiddleware.push(`Depends(${dm[1]})`)
+            break
+          }
+        }
+      }
+      if (FASTAPI_SECURITY_PATTERN.test(sig)) {
+        authMiddleware.push('Security()')
+      }
+    }
+
+    const handlerBody = handlerInfo
+      ? extractPythonFunctionBody(lines, handlerInfo.line - 1)
+      : ''
+    const inputs = detectPythonInputSources(handlerBody)
+
+    routes.push({
+      filePath,
+      methods,
+      path: normalizeFastAPIPath(path),
+      handler: handlerInfo?.name || 'unknown',
+      handlerLine: handlerInfo?.line || i + 1,
+      framework: 'fastapi',
+      authMiddleware,
+      rateLimiting: hasRateLimit,
+      isPublic: PUBLIC_ENDPOINT_PATTERN.test(path),
+      inputs,
+    })
+  }
+
+  return routes
+}
+
 // ============================================================================
 // Internal Helpers
 // ============================================================================
@@ -186,6 +313,21 @@ function findFunctionBelow(lines: string[], decoratorLine: number): { name: stri
   return null
 }
 
+/**
+ * Extract the full function signature, handling multi-line signatures.
+ * Starts from the `def` line and continues until `):`  is found or max 10 lines.
+ */
+function extractFunctionSignature(lines: string[], defLineIndex: number): string {
+  const parts: string[] = []
+  for (let i = defLineIndex; i < Math.min(defLineIndex + 10, lines.length); i++) {
+    parts.push(lines[i])
+    if (lines[i].includes('):') || lines[i].includes(') :') || lines[i].match(/\)\s*(->\s*\S+)?\s*:/)) {
+      break
+    }
+  }
+  return parts.join('\n')
+}
+
 function extractPythonFunctionBody(lines: string[], defLineIndex: number): string {
   const end = Math.min(defLineIndex + 30, lines.length)
   return lines.slice(defLineIndex, end).join('\n')
@@ -201,6 +343,11 @@ function normalizeDjangoPath(path: string): string {
   return path.replace(/<(?:\w+:)?(\w+)>/g, ':$1')
 }
 
+function normalizeFastAPIPath(path: string): string {
+  // Convert FastAPI path params {item_id} to :item_id
+  return path.replace(/\{(\w+)\}/g, ':$1')
+}
+
 function detectPythonInputSources(body: string): RouteInput[] {
   const inputs: RouteInput[] = []
   const seen = new Set<string>()

package/src/model/route-discovery/types.ts

@@ -7,7 +7,7 @@
 
 export type RouteFramework =
   | 'express' | 'nextjs_app' | 'nextjs_pages' | 'fastify'
-  | 'hono' | 'koa' | 'nestjs' | 'django' | 'flask' | 'rails'
+  | 'hono' | 'koa' | 'nestjs' | 'django' | 'flask' | 'fastapi' | 'rails'
 
 export interface RouteDefinition {
   filePath: string

package/src/model/route-discovery/utils.ts

@@ -52,3 +52,76 @@ export function extractBlock(lines: string[], startLine: number): string {
   }
   return result.join('\n')
 }
+
+/**
+ * Extract a Python block using indentation (no braces).
+ * Starts from the `def`/`async def` line and includes all lines
+ * indented deeper than the definition line.
+ * Handles blank lines, comments within the body, and multi-line signatures.
+ * Stops at a de-dented line. Max 50 lines.
+ */
+export function extractPythonBlock(lines: string[], startLine: number): string {
+  const result: string[] = []
+  if (startLine >= lines.length) return ''
+
+  const defLine = lines[startLine]
+  result.push(defLine)
+
+  // Determine the indent of the def line
+  const defIndent = defLine.search(/\S/)
+  if (defIndent < 0) return defLine
+
+  // Handle multi-line function signatures (unclosed paren on def line)
+  let bodyStart = startLine + 1
+  const hasOpenParen = defLine.includes('(')
+  const hasClosingColon = /\)\s*(?:->\s*\S+\s*)?:/.test(defLine)
+  if (hasOpenParen && !hasClosingColon) {
+    // Signature spans multiple lines — include them and find where it ends
+    for (let i = startLine + 1; i < Math.min(startLine + 10, lines.length); i++) {
+      result.push(lines[i])
+      bodyStart = i + 1
+      if (/\)\s*(?:->\s*\S+\s*)?:/.test(lines[i])) {
+        break
+      }
+    }
+  }
+
+  // Body indent is the first non-blank line after signature
+  let bodyIndent = -1
+  for (let i = bodyStart; i < Math.min(startLine + 50, lines.length); i++) {
+    const line = lines[i]
+    const trimmed = line.trim()
+    if (trimmed.length === 0) continue // skip blank lines
+    const indent = line.search(/\S/)
+    if (indent > defIndent) {
+      bodyIndent = indent
+      break
+    } else {
+      // De-dented immediately, no body
+      break
+    }
+  }
+
+  if (bodyIndent < 0) return result.join('\n')
+
+  for (let i = bodyStart; i < Math.min(startLine + 50, lines.length); i++) {
+    const line = lines[i]
+    const trimmed = line.trim()
+
+    // Blank lines and comments within body are included
+    if (trimmed.length === 0 || trimmed.startsWith('#')) {
+      result.push(line)
+      continue
+    }
+
+    const indent = line.search(/\S/)
+    if (indent >= bodyIndent) {
+      result.push(line)
+    } else {
+      // De-dented — end of block
+      break
+    }
+  }
+
+  return result.join('\n')
+}

package/src/model/taint-types.ts

@@ -129,8 +129,6 @@ export interface FileTaintAnalysis {
 import type { ProjectContext } from './project-context'
 import type { RouteMap } from './route-discovery/types'
 import type { ModuleGraph, ModuleGraphStats } from './module-graph'
-import type { CrossFileTaintPath } from './cross-file-taint'
-import type { FunctionProfileMap } from './function-classifier'
 import type { FileFrameworkAnalysis } from './framework-models/types'
 
 /**
@@ -141,9 +139,6 @@ export interface ContextEngineResult {
   projectContext: ProjectContext
   fileTaintAnalyses: Map<string, FileTaintAnalysis>
   routeMap?: RouteMap                           // CE Phase 2: Route & Handler Discovery
-  // CE Phase 4: Cross-File Analysis
-  moduleGraph?: ModuleGraph
-  crossFileTaintPaths?: CrossFileTaintPath[]
-  functionProfiles?: FunctionProfileMap
+  moduleGraph?: ModuleGraph                    // CE Phase 4: Cross-File Analysis
   frameworkAnalyses?: Map<string, FileFrameworkAnalysis>  // CE Phase 5: Framework-Aware Analysis
 }

package/src/parse/ast.ts

@@ -0,0 +1,271 @@
+/**
+ * Tree-sitter AST Integration
+ *
+ * Parses source files into typed ASTs using tree-sitter.
+ * Supports JS, TS, TSX, and JSX with language auto-detection.
+ * Includes an LRU cache keyed by content hash for incremental scan reuse.
+ */
+
+import Parser from 'tree-sitter'
+import JavaScript from 'tree-sitter-javascript'
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const { typescript: TypeScript, tsx: TSX } = require('tree-sitter-typescript')
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const Python = require('tree-sitter-python')
+import { createHash } from 'crypto'
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export type ASTLanguage = 'javascript' | 'typescript' | 'tsx' | 'python'
+
+export interface ParsedAST {
+  tree: Parser.Tree
+  language: ASTLanguage
+  filePath: string
+  /** Content hash used as cache key */
+  hash: string
+}
+
+// ============================================================================
+// Language Detection
+// ============================================================================
+
+const EXT_TO_LANGUAGE: Record<string, ASTLanguage> = {
+  '.js': 'javascript',
+  '.mjs': 'javascript',
+  '.cjs': 'javascript',
+  '.jsx': 'tsx',        // JSX uses TSX grammar (superset)
+  '.ts': 'typescript',
+  '.mts': 'typescript',
+  '.cts': 'typescript',
+  '.tsx': 'tsx',
+  '.py': 'python',
+}
+
+/**
+ * Detect AST language from file extension.
+ * Returns null for unsupported file types (Go, config files, etc.)
+ */
+export function detectLanguage(filePath: string): ASTLanguage | null {
+  const lastDot = filePath.lastIndexOf('.')
+  if (lastDot === -1) return null
+  const ext = filePath.slice(lastDot).toLowerCase()
+  return EXT_TO_LANGUAGE[ext] ?? null
+}
+
+// ============================================================================
+// Parser Pool (one per language to avoid setLanguage churn)
+// ============================================================================
+
+const parsers = new Map<ASTLanguage, Parser>()
+
+function getParser(language: ASTLanguage): Parser {
+  let parser = parsers.get(language)
+  if (parser) return parser
+
+  parser = new Parser()
+  switch (language) {
+    case 'javascript':
+      parser.setLanguage(JavaScript)
+      break
+    case 'typescript':
+      parser.setLanguage(TypeScript)
+      break
+    case 'tsx':
+      parser.setLanguage(TSX)
+      break
+    case 'python':
+      parser.setLanguage(Python)
+      break
+  }
+  parsers.set(language, parser)
+  return parser
+}
+
+// ============================================================================
+// AST Cache (LRU by content hash)
+// ============================================================================
+
+const MAX_CACHE_SIZE = 500
+
+interface CacheEntry {
+  ast: ParsedAST
+  lastAccess: number
+}
+
+const astCache = new Map<string, CacheEntry>()
+
+function hashContent(content: string): string {
+  return createHash('sha1').update(content).digest('hex')
+}
+
+function evictIfNeeded(): void {
+  if (astCache.size <= MAX_CACHE_SIZE) return
+
+  // Evict least recently accessed
+  let oldest: string | null = null
+  let oldestTime = Infinity
+  for (const [key, entry] of astCache) {
+    if (entry.lastAccess < oldestTime) {
+      oldestTime = entry.lastAccess
+      oldest = key
+    }
+  }
+  if (oldest) astCache.delete(oldest)
+}
+
+// ============================================================================
+// Public API
+// ============================================================================
+
+/**
+ * Parse a file into a tree-sitter AST.
+ * Returns null if the language is unsupported or parsing fails.
+ * Results are cached by content hash for incremental scan reuse.
+ */
+export function parseFile(content: string, filePath: string): ParsedAST | null {
+  const language = detectLanguage(filePath)
+  if (!language) return null
+
+  const hash = hashContent(content)
+  const cacheKey = `${language}:${hash}`
+
+  // Check cache
+  const cached = astCache.get(cacheKey)
+  if (cached) {
+    cached.lastAccess = Date.now()
+    // Return with correct filePath (cache may hold AST from a different path with same content)
+    if (cached.ast.filePath !== filePath) {
+      return { ...cached.ast, filePath }
+    }
+    return cached.ast
+  }
+
+  // Parse
+  try {
+    const parser = getParser(language)
+    const tree = parser.parse(content)
+
+    const ast: ParsedAST = { tree, language, filePath, hash }
+
+    // Cache
+    evictIfNeeded()
+    astCache.set(cacheKey, { ast, lastAccess: Date.now() })
+
+    return ast
+  } catch {
+    // Graceful fallback — unparseable files are handled by regex detectors
+    return null
+  }
+}
+
+/**
+ * Parse multiple files, returning a map of filePath → ParsedAST.
+ * Files with unsupported languages or parse errors are omitted.
+ */
+export function parseFiles(
+  files: Array<{ path: string; content: string }>
+): Map<string, ParsedAST> {
+  const results = new Map<string, ParsedAST>()
+  for (const file of files) {
+    const ast = parseFile(file.content, file.path)
+    if (ast) {
+      results.set(file.path, ast)
+    }
+  }
+  return results
+}
+
+/**
+ * Clear the AST cache. Useful for testing.
+ */
+export function clearASTCache(): void {
+  astCache.clear()
+}
+
+/**
+ * Get current cache size. Useful for diagnostics.
+ */
+export function getASTCacheSize(): number {
+  return astCache.size
+}
+
+// ============================================================================
+// AST Traversal Helpers
+// ============================================================================
+
+/**
+ * Walk all descendants of a node, calling visitor for each.
+ * Uses a cursor for efficient traversal (avoids creating child arrays).
+ */
+export function walkAST(
+  node: Parser.SyntaxNode,
+  visitor: (node: Parser.SyntaxNode) => void | boolean
+): void {
+  const cursor = node.walk()
+  let reachedRoot = false
+
+  while (!reachedRoot) {
+    // Visit current node; if visitor returns true, skip subtree
+    const skip = visitor(cursor.currentNode)
+
+    if (!skip && cursor.gotoFirstChild()) {
+      continue
+    }
+
+    if (cursor.gotoNextSibling()) {
+      continue
+    }
+
+    // Walk back up until we can go to a sibling
+    while (true) {
+      if (!cursor.gotoParent()) {
+        reachedRoot = true
+        break
+      }
+      if (cursor.gotoNextSibling()) {
+        break
+      }
+    }
+  }
+}
+
+/**
+ * Find all AST nodes matching a given type.
+ * Equivalent to `node.descendantsOfType(type)` but with better typing.
+ */
+export function findNodesOfType(
+  root: Parser.SyntaxNode,
+  type: string | string[]
+): Parser.SyntaxNode[] {
+  const types = Array.isArray(type) ? type : [type]
+  return root.descendantsOfType(types)
+}
+
+/**
+ * Find the innermost function or method surrounding a given node.
+ */
+export function findEnclosingFunction(node: Parser.SyntaxNode): Parser.SyntaxNode | null {
+  const functionTypes = [
+    // JavaScript/TypeScript
+    'function_declaration',
+    'function_expression',
+    'arrow_function',
+    'method_definition',
+    'generator_function_declaration',
+    'generator_function',
+    // Python
+    'function_definition',
+  ]
+
+  let current = node.parent
+  while (current) {
+    if (functionTypes.includes(current.type)) {
+      return current
+    }
+    current = current.parent
+  }
+  return null
+}