driftdetect 0.4.6 → 0.5.0

package/dist/services/scanner-service.js

@@ -1,33 +1,28 @@
 /**
- * Scanner Service - Enterprise-grade pattern detection
+ * Scanner Service - Enterprise-grade pattern detection with Worker Threads
  *
  * Uses the real detectors from driftdetect-detectors to find
  * high-value architectural patterns and violations.
  *
- * Now includes manifest generation for pattern location discovery.
+ * Now uses Piscina worker threads for parallel CPU-bound processing.
  */
 import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
+import * as os from 'node:os';
+import { fileURLToPath } from 'node:url';
 import { createAllDetectorsArray, getDetectorCounts, } from 'driftdetect-detectors';
 import { ManifestStore, hashContent, } from 'driftdetect-core';
+// Get the directory of this module for worker path resolution
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
 // ============================================================================
 // Location Deduplication
 // ============================================================================
-/**
- * Create a unique key for a basic location to enable deduplication
- */
 function locationKey(loc) {
     return `${loc.file}:${loc.line}:${loc.column}`;
 }
-/**
- * Create a unique key for a semantic location to enable deduplication
- */
 function semanticLocationKey(loc) {
     return `${loc.file}:${loc.range.start}:${loc.range.end}:${loc.name}`;
 }
-/**
- * Add a basic location to an array only if it doesn't already exist
- */
 function addUniqueLocation(locations, location, seenKeys) {
     const key = locationKey(location);
     const seen = seenKeys || new Set(locations.map(locationKey));
@@ -38,9 +33,6 @@ function addUniqueLocation(locations, location, seenKeys) {
     locations.push(location);
     return true;
 }
-/**
- * Add a semantic location to an array only if it doesn't already exist
- */
 function addUniqueSemanticLocation(locations, location, seenKeys) {
     const key = semanticLocationKey(location);
     const seen = seenKeys || new Set(locations.map(semanticLocationKey));
@@ -87,32 +79,25 @@ function isDetectorApplicable(detector, language) {
     return info.supportedLanguages.includes(language);
 }
 // ============================================================================
-// Critical Detectors (highest value)
+// Critical Detectors
 // ============================================================================
 const CRITICAL_DETECTOR_IDS = new Set([
-    // Security - CRITICAL
     'security/sql-injection',
     'security/xss-prevention',
     'security/secret-management',
     'security/input-sanitization',
     'security/csrf-protection',
-    // Auth - CRITICAL
     'auth/middleware-usage',
     'auth/token-handling',
-    // API - HIGH
     'api/route-structure',
     'api/error-format',
     'api/response-envelope',
-    // Data Access - HIGH
     'data-access/n-plus-one',
     'data-access/query-patterns',
-    // Structural - HIGH
     'structural/circular-deps',
     'structural/module-boundaries',
-    // Errors - HIGH
     'errors/exception-hierarchy',
     'errors/try-catch-placement',
-    // Logging - HIGH
     'logging/pii-redaction',
 ]);
 // ============================================================================
@@ -121,27 +106,34 @@ const CRITICAL_DETECTOR_IDS = new Set([
 /**
  * Scanner Service
  *
- * Orchestrates pattern detection across files using real detectors
- * from driftdetect-detectors package.
+ * Orchestrates pattern detection across files using real detectors.
+ * Supports both single-threaded and multi-threaded (Piscina) modes.
  */
 export class ScannerService {
     config;
     detectors = [];
     initialized = false;
     manifestStore = null;
+    pool = null;
+    PiscinaClass = null;
     constructor(config) {
-        this.config = config;
+        this.config = {
+            ...config,
+            // Default to using worker threads
+            useWorkerThreads: config.useWorkerThreads ?? true,
+            workerThreads: config.workerThreads ?? Math.max(1, os.cpus().length - 1),
+        };
         if (config.generateManifest) {
             this.manifestStore = new ManifestStore(config.rootDir);
         }
     }
     /**
-     * Initialize the scanner service - loads all detectors
+     * Initialize the scanner service
      */
     async initialize() {
         if (this.initialized)
             return;
-        // Create all detectors
+        // Create all detectors (needed for counts even in worker mode)
         this.detectors = await createAllDetectorsArray();
         // Filter by category if specified
         if (this.config.categories && this.config.categories.length > 0) {
@@ -155,12 +147,71 @@ export class ScannerService {
         if (this.config.criticalOnly) {
             this.detectors = this.detectors.filter(d => CRITICAL_DETECTOR_IDS.has(d.id));
         }
+        // Initialize worker pool if using threads
+        if (this.config.useWorkerThreads) {
+            try {
+                const piscinaModule = await import('piscina');
+                // Piscina exports as named export, not default
+                this.PiscinaClass = (piscinaModule.Piscina || piscinaModule.default);
+                if (!this.PiscinaClass || typeof this.PiscinaClass !== 'function') {
+                    throw new Error(`Piscina class not found. Module exports: ${Object.keys(piscinaModule).join(', ')}`);
+                }
+                // Worker path - compiled JS in dist
+                const workerPath = path.join(__dirname, '..', 'workers', 'detector-worker.js');
+                if (this.config.verbose) {
+                    console.log(`  Worker path: ${workerPath}`);
+                }
+                this.pool = new this.PiscinaClass({
+                    filename: workerPath,
+                    minThreads: this.config.workerThreads, // Keep all workers alive
+                    maxThreads: this.config.workerThreads,
+                    idleTimeout: 60000, // 60s idle timeout
+                });
+                if (this.config.verbose) {
+                    console.log(`  Worker pool initialized with ${this.config.workerThreads} threads`);
+                }
+                // Warm up all workers in parallel - this loads detectors once per worker
+                await this.warmupWorkers();
+            }
+            catch (error) {
+                // Fall back to single-threaded mode
+                console.log(`  Worker threads unavailable, using single-threaded mode: ${error.message}`);
+                if (this.config.verbose) {
+                    console.log(`  Full error: ${error.stack}`);
+                }
+                this.config.useWorkerThreads = false;
+            }
+        }
         // Load existing manifest for incremental scanning
         if (this.manifestStore) {
             await this.manifestStore.load();
         }
         this.initialized = true;
     }
+    /**
+     * Warm up all worker threads by preloading detectors
+     * This runs warmup tasks in parallel so all workers load detectors simultaneously
+     */
+    async warmupWorkers() {
+        if (!this.pool)
+            return;
+        const numWorkers = this.config.workerThreads;
+        const warmupTasks = Array(numWorkers).fill(null).map(() => ({
+            type: 'warmup',
+            categories: this.config.categories,
+            criticalOnly: this.config.criticalOnly,
+        }));
+        if (this.config.verbose) {
+            console.log(`  Warming up ${numWorkers} workers...`);
+        }
+        const startTime = Date.now();
+        // Run warmup tasks in parallel - each worker gets one task
+        await Promise.all(warmupTasks.map(task => this.pool.run(task)));
+        const duration = Date.now() - startTime;
+        if (this.config.verbose) {
+            console.log(`  Workers warmed up in ${duration}ms`);
+        }
+    }
     /**
      * Get detector count
      */
@@ -174,16 +225,211 @@ export class ScannerService {
         return getDetectorCounts();
     }
     /**
-     * Scan files for patterns using real detectors
+     * Check if worker threads are enabled and initialized
+     */
+    isUsingWorkerThreads() {
+        return this.config.useWorkerThreads === true && this.pool !== null;
+    }
+    /**
+     * Get worker thread count
+     */
+    getWorkerThreadCount() {
+        return this.config.workerThreads ?? 0;
+    }
+    /**
+     * Scan files for patterns
      */
     async scanFiles(files, projectContext) {
+        if (this.config.useWorkerThreads && this.pool) {
+            return this.scanFilesWithWorkers(files, projectContext);
+        }
+        return this.scanFilesSingleThreaded(files, projectContext);
+    }
+    /**
+     * Scan files using worker threads (parallel)
+     */
+    async scanFilesWithWorkers(files, projectContext) {
+        const startTime = Date.now();
+        const errors = [];
+        // Filter to changed files if incremental
+        let filesToScan = files;
+        if (this.config.incremental && this.manifestStore) {
+            const fullPaths = files.map(f => path.join(this.config.rootDir, f));
+            const changedPaths = await this.manifestStore.getChangedFiles(fullPaths);
+            filesToScan = changedPaths.map(f => path.relative(this.config.rootDir, f));
+            for (const file of filesToScan) {
+                this.manifestStore.clearFilePatterns(path.join(this.config.rootDir, file));
+            }
+        }
+        // Create tasks for worker pool
+        const tasks = filesToScan.map(file => ({
+            file,
+            rootDir: this.config.rootDir,
+            projectFiles: projectContext.files,
+            projectConfig: projectContext.config,
+            categories: this.config.categories,
+            criticalOnly: this.config.criticalOnly,
+        }));
+        // Run all tasks in parallel
+        const results = await Promise.all(tasks.map(async (task) => {
+            try {
+                return await this.pool.run(task);
+            }
+            catch (error) {
+                return {
+                    file: task.file,
+                    language: null,
+                    patterns: [],
+                    violations: [],
+                    detectorsRan: 0,
+                    detectorsSkipped: 0,
+                    duration: 0,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        }));
+        // Aggregate results
+        return this.aggregateWorkerResults(results, files, startTime, errors);
+    }
+    /**
+     * Aggregate results from worker threads
+     */
+    async aggregateWorkerResults(results, allFiles, startTime, errors) {
+        const fileResults = [];
+        const patternMap = new Map();
+        const allViolations = [];
+        const manifestPatternMap = new Map();
+        let totalDetectorsRan = 0;
+        let totalDetectorsSkipped = 0;
+        for (const result of results) {
+            if (result.error) {
+                errors.push(`Failed to scan ${result.file}: ${result.error}`);
+            }
+            totalDetectorsRan += result.detectorsRan;
+            totalDetectorsSkipped += result.detectorsSkipped;
+            // Convert to FileScanResult
+            const filePatterns = result.patterns.map(p => ({
+                patternId: p.patternId,
+                detectorId: p.detectorId,
+                confidence: p.confidence,
+                location: p.location,
+            }));
+            fileResults.push({
+                file: result.file,
+                patterns: filePatterns,
+                violations: result.violations,
+                duration: result.duration,
+                error: result.error,
+            });
+            // Aggregate patterns
+            for (const match of result.patterns) {
+                const key = match.patternId;
+                const existing = patternMap.get(key);
+                if (existing) {
+                    addUniqueLocation(existing.locations, match.location);
+                    existing.occurrences++;
+                    existing.confidence = Math.max(existing.confidence, match.confidence);
+                }
+                else {
+                    patternMap.set(key, {
+                        patternId: match.patternId,
+                        detectorId: match.detectorId,
+                        category: match.category,
+                        subcategory: match.subcategory,
+                        name: match.detectorName,
+                        description: match.detectorDescription,
+                        locations: [match.location],
+                        confidence: match.confidence,
+                        occurrences: 1,
+                    });
+                }
+                // Build manifest pattern
+                if (this.config.generateManifest && result.language) {
+                    await this.addToManifest(manifestPatternMap, match, result);
+                }
+            }
+            // Aggregate violations
+            for (const violation of result.violations) {
+                allViolations.push(violation);
+            }
+        }
+        // Convert pattern map to array
+        const patterns = Array.from(patternMap.values());
+        // Build and save manifest
+        let manifest;
+        if (this.config.generateManifest && this.manifestStore) {
+            const manifestPatterns = Array.from(manifestPatternMap.values());
+            this.manifestStore.updatePatterns(manifestPatterns);
+            await this.manifestStore.save();
+            manifest = await this.manifestStore.get();
+        }
+        return {
+            files: fileResults,
+            patterns,
+            violations: allViolations,
+            totalPatterns: patterns.reduce((sum, p) => sum + p.occurrences, 0),
+            totalViolations: allViolations.length,
+            totalFiles: allFiles.length,
+            duration: Date.now() - startTime,
+            errors,
+            detectorStats: {
+                total: this.detectors.length,
+                ran: totalDetectorsRan,
+                skipped: totalDetectorsSkipped,
+            },
+            workerStats: this.pool ? {
+                threadsUsed: this.pool.threads.length,
+                tasksCompleted: this.pool.completed,
+            } : undefined,
+            manifest,
+        };
+    }
+    /**
+     * Add pattern match to manifest
+     */
+    async addToManifest(manifestPatternMap, match, result) {
+        try {
+            const filePath = path.join(this.config.rootDir, result.file);
+            const content = await fs.readFile(filePath, 'utf-8');
+            const contentHash = hashContent(content);
+            const language = result.language;
+            const semanticLoc = this.createSemanticLocation(match.location, content, contentHash, language);
+            const manifestKey = `${match.category}/${match.subcategory}/${match.patternId}`;
+            const existingManifest = manifestPatternMap.get(manifestKey);
+            if (existingManifest) {
+                addUniqueSemanticLocation(existingManifest.locations, semanticLoc);
+                existingManifest.confidence = Math.max(existingManifest.confidence, match.confidence);
+                existingManifest.lastSeen = new Date().toISOString();
+            }
+            else {
+                manifestPatternMap.set(manifestKey, {
+                    id: manifestKey,
+                    name: match.detectorName,
+                    category: match.category,
+                    subcategory: match.subcategory,
+                    status: 'discovered',
+                    confidence: match.confidence,
+                    locations: [semanticLoc],
+                    outliers: [],
+                    description: match.detectorDescription,
+                    firstSeen: new Date().toISOString(),
+                    lastSeen: new Date().toISOString(),
+                });
+            }
+        }
+        catch {
+            // Ignore manifest errors
+        }
+    }
+    /**
+     * Scan files single-threaded (fallback)
+     */
+    async scanFilesSingleThreaded(files, projectContext) {
         const startTime = Date.now();
         const fileResults = [];
         const errors = [];
-        // Aggregation maps
         const patternMap = new Map();
         const allViolations = [];
-        // Manifest pattern aggregation
         const manifestPatternMap = new Map();
         let detectorsRan = 0;
         let detectorsSkipped = 0;
@@ -193,7 +439,6 @@ export class ScannerService {
             const fullPaths = files.map(f => path.join(this.config.rootDir, f));
             const changedPaths = await this.manifestStore.getChangedFiles(fullPaths);
             filesToScan = changedPaths.map(f => path.relative(this.config.rootDir, f));
-            // Clear patterns for changed files
             for (const file of filesToScan) {
                 this.manifestStore.clearFilePatterns(path.join(this.config.rootDir, file));
             }
@@ -202,7 +447,6 @@ export class ScannerService {
             const fileStart = Date.now();
             const filePath = path.join(this.config.rootDir, file);
             const language = getLanguage(file);
-            // Skip files we can't detect language for
             if (!language) {
                 fileResults.push({
                     file,
@@ -217,14 +461,13 @@ export class ScannerService {
                 const contentHash = hashContent(content);
                 const filePatterns = [];
                 const fileViolations = [];
-                // Create detection context
                 const context = {
                     file,
                     content,
                     language,
-                    ast: null, // AST parsing is optional
-                    imports: [], // Will be populated by detectors that need it
-                    exports: [], // Will be populated by detectors that need it
+                    ast: null,
+                    imports: [],
+                    exports: [],
                     extension: path.extname(file),
                     isTestFile: /\.(test|spec)\.[jt]sx?$/.test(file) || file.includes('__tests__'),
                     isTypeDefinition: file.endsWith('.d.ts'),
@@ -234,7 +477,6 @@ export class ScannerService {
                         config: projectContext.config,
                     },
                 };
-                // Run applicable detectors
                 for (const detector of this.detectors) {
                     if (!isDetectorApplicable(detector, language)) {
                         detectorsSkipped++;
@@ -244,7 +486,6 @@ export class ScannerService {
                     try {
                         const result = await detector.detect(context);
                         const info = detector.getInfo();
-                        // Process patterns
                         for (const match of result.patterns) {
                             filePatterns.push({
                                 patternId: match.patternId,
@@ -252,7 +493,6 @@ export class ScannerService {
                                 confidence: match.confidence,
                                 location: match.location,
                             });
-                            // Aggregate pattern
                             const key = match.patternId;
                             const existing = patternMap.get(key);
                             if (existing) {
@@ -273,7 +513,6 @@ export class ScannerService {
                                     occurrences: 1,
                                 });
                             }
-                            // Build manifest pattern with semantic location
                             if (this.config.generateManifest) {
                                 const semanticLoc = this.createSemanticLocation(match.location, content, contentHash, language);
                                 const manifestKey = `${info.category}/${info.subcategory}/${match.patternId}`;
@@ -300,17 +539,14 @@ export class ScannerService {
                                 }
                             }
                         }
-                        // Process violations
-                        // First check the standard violations array
+                        // Process violations (same as before)
                         let violationsToProcess = result.violations;
-                        // If empty, check for violations in custom metadata (many detectors store them there)
                         if (violationsToProcess.length === 0 && result.metadata?.custom) {
                             const customData = result.metadata.custom;
                             const customViolations = customData['violations'];
                             if (customViolations && Array.isArray(customViolations)) {
-                                // Convert custom violations to standard format
                                 violationsToProcess = customViolations.map(cv => {
-                                    const violation = {
+                                    const v = {
                                         id: `${detector.id}-${cv.file}-${cv.line}-${cv.column}`,
                                         patternId: detector.id,
                                         severity: cv.severity || 'warning',
@@ -328,9 +564,9 @@ export class ScannerService {
                                         occurrences: 1,
                                     };
                                     if (cv.type) {
-                                        violation.explanation = `Violation type: ${cv.type}`;
+                                        v.explanation = `Violation type: ${cv.type}`;
                                     }
-                                    return violation;
+                                    return v;
                                 });
                             }
                         }
@@ -344,39 +580,13 @@ export class ScannerService {
                                 line: violation.range.start.line + 1,
                                 column: violation.range.start.character + 1,
                                 message: violation.message,
-                                explanation: violation.explanation,
                                 suggestedFix: violation.expected,
                             };
+                            if (violation.explanation) {
+                                aggViolation.explanation = violation.explanation;
+                            }
                             fileViolations.push(aggViolation);
                             allViolations.push(aggViolation);
-                            // Add violation as outlier to manifest
-                            if (this.config.generateManifest) {
-                                const outlierLoc = this.createSemanticLocation({
-                                    file: violation.file,
-                                    line: violation.range.start.line + 1,
-                                    column: violation.range.start.character + 1,
-                                }, content, contentHash, language, violation.message);
-                                const manifestKey = `${info.category}/${info.subcategory}/${violation.patternId}`;
-                                const existingManifest = manifestPatternMap.get(manifestKey);
-                                if (existingManifest) {
-                                    existingManifest.outliers.push(outlierLoc);
-                                }
-                                else {
-                                    manifestPatternMap.set(manifestKey, {
-                                        id: manifestKey,
-                                        name: info.name,
-                                        category: info.category,
-                                        subcategory: info.subcategory,
-                                        status: 'discovered',
-                                        confidence: 0.5,
-                                        locations: [],
-                                        outliers: [outlierLoc],
-                                        description: info.description,
-                                        firstSeen: new Date().toISOString(),
-                                        lastSeen: new Date().toISOString(),
-                                    });
-                                }
-                            }
                         }
                     }
                     catch (detectorError) {
@@ -404,9 +614,7 @@ export class ScannerService {
                 });
             }
         }
-        // Convert pattern map to array
         const patterns = Array.from(patternMap.values());
-        // Build and save manifest
         let manifest;
         if (this.config.generateManifest && this.manifestStore) {
             const manifestPatterns = Array.from(manifestPatternMap.values());
@@ -414,7 +622,7 @@ export class ScannerService {
             await this.manifestStore.save();
             manifest = await this.manifestStore.get();
         }
-        const result = {
+        return {
             files: fileResults,
             patterns,
             violations: allViolations,
@@ -428,11 +636,8 @@ export class ScannerService {
                 ran: detectorsRan,
                 skipped: detectorsSkipped,
             },
+            manifest,
         };
-        if (manifest) {
-            result.manifest = manifest;
-        }
-        return result;
     }
     /**
      * Create a semantic location from a basic location
@@ -440,7 +645,6 @@ export class ScannerService {
     createSemanticLocation(location, content, hash, language, name) {
         const lines = content.split('\n');
         const lineContent = lines[location.line - 1] || '';
-        // Try to extract semantic info from the line
         const semanticInfo = this.extractSemanticInfo(lineContent, language);
         const result = {
             file: location.file,
@@ -465,89 +669,81 @@ export class ScannerService {
      */
     extractSemanticInfo(line, language) {
         const trimmed = line.trim();
-        // TypeScript/JavaScript patterns
         if (language === 'typescript' || language === 'javascript') {
-            // Class
             const classMatch = trimmed.match(/^(?:export\s+)?(?:abstract\s+)?class\s+(\w+)/);
             if (classMatch && classMatch[1]) {
                 return { type: 'class', name: classMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // Function
             const funcMatch = trimmed.match(/^(?:export\s+)?(?:async\s+)?function\s+(\w+)/);
             if (funcMatch && funcMatch[1]) {
                 return { type: 'function', name: funcMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // Arrow function / const
             const arrowMatch = trimmed.match(/^(?:export\s+)?const\s+(\w+)\s*=/);
             if (arrowMatch && arrowMatch[1]) {
                 return { type: 'function', name: arrowMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // Interface
             const interfaceMatch = trimmed.match(/^(?:export\s+)?interface\s+(\w+)/);
             if (interfaceMatch && interfaceMatch[1]) {
                 return { type: 'interface', name: interfaceMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // Type
             const typeMatch = trimmed.match(/^(?:export\s+)?type\s+(\w+)/);
             if (typeMatch && typeMatch[1]) {
                 return { type: 'type', name: typeMatch[1], signature: trimmed.substring(0, 80) };
             }
         }
-        // Python patterns
         if (language === 'python') {
-            // Class
             const classMatch = trimmed.match(/^class\s+(\w+)/);
             if (classMatch && classMatch[1]) {
                 return { type: 'class', name: classMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // Function/method
             const defMatch = trimmed.match(/^(?:async\s+)?def\s+(\w+)/);
             if (defMatch && defMatch[1]) {
                 return { type: 'function', name: defMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // Decorator
             const decoratorMatch = trimmed.match(/^@(\w+)/);
             if (decoratorMatch && decoratorMatch[1]) {
                 return { type: 'decorator', name: decoratorMatch[1], signature: trimmed };
             }
         }
-        // PHP patterns
         if (language === 'php') {
-            // Class
             const classMatch = trimmed.match(/^(?:abstract\s+|final\s+)?class\s+(\w+)/);
             if (classMatch && classMatch[1]) {
                 return { type: 'class', name: classMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // Interface
             const interfaceMatch = trimmed.match(/^interface\s+(\w+)/);
             if (interfaceMatch && interfaceMatch[1]) {
                 return { type: 'interface', name: interfaceMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // Trait
             const traitMatch = trimmed.match(/^trait\s+(\w+)/);
             if (traitMatch && traitMatch[1]) {
                 return { type: 'class', name: traitMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // Function/method
             const funcMatch = trimmed.match(/^(?:public|protected|private)?\s*(?:static\s+)?function\s+(\w+)/);
             if (funcMatch && funcMatch[1]) {
                 return { type: 'function', name: funcMatch[1], signature: trimmed.substring(0, 80) };
             }
-            // PHP 8 Attribute
             const attrMatch = trimmed.match(/^#\[(\w+)/);
             if (attrMatch && attrMatch[1]) {
                 return { type: 'decorator', name: attrMatch[1], signature: trimmed };
             }
         }
-        // Default to block
         return { type: 'block' };
     }
     /**
-     * Get the manifest store (for external access)
+     * Get the manifest store
      */
     getManifestStore() {
         return this.manifestStore;
     }
+    /**
+     * Destroy the worker pool
+     */
+    async destroy() {
+        if (this.pool) {
+            await this.pool.destroy();
+            this.pool = null;
+        }
+    }
 }
 /**
  * Create a scanner service