honeyweb-core 1.0.3 → 2.0.0

package/detection/bot-detector.js

@@ -0,0 +1,140 @@
+// honeyweb-core/detection/bot-detector.js
+// Bot fingerprinting - detect automated tools and headless browsers
+
+class BotDetector {
+    constructor(config = {}) {
+        this.enabled = config.enabled !== false;
+    }
+
+    /**
+     * Detect if request is from a bot
+     * @param {Object} req - Express request object
+     * @returns {Object} - { isBot: boolean, confidence: number, indicators: string[] }
+     */
+    detect(req) {
+        if (!this.enabled) {
+            return { isBot: false, confidence: 0, indicators: [] };
+        }
+
+        const indicators = [];
+        let confidence = 0;
+
+        const userAgent = req.headers['user-agent'] || '';
+        const headers = req.headers;
+
+        // 1. USER-AGENT ANALYSIS
+
+        // Known bot patterns
+        const knownBotPatterns = [
+            { pattern: /curl/i, name: 'curl', confidence: 90 },
+            { pattern: /wget/i, name: 'wget', confidence: 90 },
+            { pattern: /python-requests/i, name: 'Python Requests', confidence: 85 },
+            { pattern: /scrapy/i, name: 'Scrapy', confidence: 90 },
+            { pattern: /selenium/i, name: 'Selenium', confidence: 80 },
+            { pattern: /puppeteer/i, name: 'Puppeteer', confidence: 85 },
+            { pattern: /phantomjs/i, name: 'PhantomJS', confidence: 85 },
+            { pattern: /headlesschrome/i, name: 'Headless Chrome', confidence: 80 },
+            { pattern: /bot|crawler|spider|scraper/i, name: 'Generic Bot', confidence: 70 },
+            { pattern: /axios/i, name: 'Axios', confidence: 75 },
+            { pattern: /node-fetch/i, name: 'Node Fetch', confidence: 75 },
+            { pattern: /go-http-client/i, name: 'Go HTTP Client', confidence: 80 },
+            { pattern: /java/i, name: 'Java Client', confidence: 60 },
+            { pattern: /okhttp/i, name: 'OkHttp', confidence: 75 }
+        ];
+
+        for (const { pattern, name, confidence: conf } of knownBotPatterns) {
+            if (pattern.test(userAgent)) {
+                indicators.push(`Known bot User-Agent: ${name}`);
+                confidence = Math.max(confidence, conf);
+            }
+        }
+
+        // Empty or missing User-Agent
+        if (!userAgent || userAgent.trim() === '') {
+            indicators.push('Missing User-Agent header');
+            confidence = Math.max(confidence, 70);
+        }
+
+        // 2. HEADER ANALYSIS
+
+        // Missing common browser headers
+        const commonHeaders = ['accept', 'accept-language', 'accept-encoding'];
+        const missingHeaders = commonHeaders.filter(h => !headers[h]);
+
+        if (missingHeaders.length > 0) {
+            indicators.push(`Missing headers: ${missingHeaders.join(', ')}`);
+            confidence = Math.max(confidence, 40 + (missingHeaders.length * 10));
+        }
+
+        // Suspicious header combinations
+        if (headers['accept'] && headers['accept'] === '*/*') {
+            indicators.push('Generic Accept header (*/*)')
+            confidence = Math.max(confidence, 30);
+        }
+
+        // No Accept-Language (browsers always send this)
+        if (!headers['accept-language']) {
+            indicators.push('Missing Accept-Language header');
+            confidence = Math.max(confidence, 40);
+        }
+
+        // Connection: close (common in automated tools)
+        if (headers['connection'] === 'close') {
+            indicators.push('Connection: close (bot pattern)');
+            confidence = Math.max(confidence, 20);
+        }
+
+        // 3. BROWSER VERSION ANALYSIS
+
+        // Detect outdated browser versions (bots often use old UA strings)
+        const chromeMatch = userAgent.match(/Chrome\/(\d+)/);
+        if (chromeMatch) {
+            const version = parseInt(chromeMatch[1]);
+            if (version < 90) {
+                indicators.push(`Outdated Chrome version: ${version}`);
+                confidence = Math.max(confidence, 30);
+            }
+        }
+
+        // 4. SUSPICIOUS PATTERNS
+
+        // User-Agent too short (< 20 chars)
+        if (userAgent.length > 0 && userAgent.length < 20) {
+            indicators.push('Suspiciously short User-Agent');
+            confidence = Math.max(confidence, 50);
+        }
+
+        // User-Agent too long (> 500 chars) - sometimes bots add extra info
+        if (userAgent.length > 500) {
+            indicators.push('Suspiciously long User-Agent');
+            confidence = Math.max(confidence, 30);
+        }
+
+        // Multiple spaces in User-Agent (malformed)
+        if (/\s{2,}/.test(userAgent)) {
+            indicators.push('Malformed User-Agent (multiple spaces)');
+            confidence = Math.max(confidence, 40);
+        }
+
+        // Cap confidence at 100
+        confidence = Math.min(100, confidence);
+
+        return {
+            isBot: confidence >= 60,
+            confidence,
+            indicators
+        };
+    }
+
+    /**
+     * Get statistics
+     * @returns {Object}
+     */
+    getStats() {
+        return {
+            enabled: this.enabled
+        };
+    }
+}
+
+module.exports = BotDetector;

package/detection/index.js

@@ -0,0 +1,136 @@
+// honeyweb-core/detection/index.js
+// Detection orchestrator - combines all detection modules
+
+const { detectMaliciousPatterns } = require('./patterns');
+const RateLimiter = require('./rate-limiter');
+const BotWhitelist = require('./whitelist');
+const BehavioralAnalyzer = require('./behavioral');
+const BotDetector = require('./bot-detector');
+
+class DetectionEngine {
+    constructor(config) {
+        this.config = config;
+
+        // Initialize rate limiter
+        this.rateLimiter = config.rateLimit.enabled
+            ? new RateLimiter(config.rateLimit)
+            : null;
+
+        // Initialize Phase 2 detection modules
+        this.whitelist = config.detection.whitelist.enabled
+            ? new BotWhitelist(config.detection.whitelist)
+            : null;
+
+        this.behavioral = config.detection.behavioral.enabled
+            ? new BehavioralAnalyzer(config.detection.behavioral)
+            : null;
+
+        this.botDetector = new BotDetector({ enabled: true });
+    }
+
+    /**
+     * Analyze request for threats
+     * @param {Object} req - Express request object
+     * @param {string} ip - Client IP address
+     * @returns {Promise<Object>} - { detected: boolean, threats: string[], threatLevel: number, whitelist: Object, behavioral: Object, botDetection: Object }
+     */
+    async analyze(req, ip) {
+        const threats = [];
+        let threatLevel = 0;
+
+        // 0. Check whitelist first (legitimate bots should skip other checks)
+        let whitelistResult = null;
+        if (this.whitelist) {
+            whitelistResult = await this.whitelist.check(req, ip);
+            if (whitelistResult.isLegitimate) {
+                // Legitimate bot - skip all other checks
+                return {
+                    detected: false,
+                    threats: [],
+                    threatLevel: 0,
+                    whitelist: whitelistResult,
+                    legitimateBot: true
+                };
+            }
+        }
+
+        // 1. Pattern detection (SQLi/XSS)
+        if (this.config.detection.patterns.enabled) {
+            const patternResult = detectMaliciousPatterns(req);
+            if (patternResult.detected) {
+                threats.push(...patternResult.threats);
+                threatLevel += 50; // High threat
+            }
+        }
+
+        // 2. Rate limiting
+        let rateLimitResult = null;
+        if (this.rateLimiter) {
+            rateLimitResult = this.rateLimiter.check(ip);
+            if (rateLimitResult.limited) {
+                threats.push(`Rate limit exceeded: ${rateLimitResult.count} requests in ${this.config.rateLimit.window}ms`);
+                threatLevel += 30; // Medium threat
+            }
+        }
+
+        // 3. Behavioral analysis (Phase 2)
+        let behavioralResult = null;
+        if (this.behavioral) {
+            behavioralResult = this.behavioral.analyze(req, ip);
+            if (behavioralResult.suspicious) {
+                threats.push(...behavioralResult.reasons);
+                threatLevel += behavioralResult.suspicionScore * 0.3; // Scale down behavioral score
+            }
+        }
+
+        // 4. Bot detection (Phase 2)
+        let botDetectionResult = null;
+        if (this.botDetector) {
+            botDetectionResult = this.botDetector.detect(req);
+            if (botDetectionResult.isBot) {
+                threats.push(...botDetectionResult.indicators);
+                threatLevel += botDetectionResult.confidence * 0.2; // Scale down bot detection score
+            }
+        }
+
+        // Cap threat level at 100
+        threatLevel = Math.min(100, threatLevel);
+
+        return {
+            detected: threats.length > 0,
+            threats,
+            threatLevel,
+            rateLimit: rateLimitResult,
+            whitelist: whitelistResult,
+            behavioral: behavioralResult,
+            botDetection: botDetectionResult,
+            legitimateBot: false
+        };
+    }
+
+    /**
+     * Get detection statistics
+     * @returns {Object}
+     */
+    getStats() {
+        return {
+            rateLimiter: this.rateLimiter ? this.rateLimiter.getStats() : null,
+            behavioral: this.behavioral ? this.behavioral.getStats() : null,
+            botDetector: this.botDetector ? this.botDetector.getStats() : null
+        };
+    }
+
+    /**
+     * Cleanup and stop timers
+     */
+    destroy() {
+        if (this.rateLimiter) {
+            this.rateLimiter.destroy();
+        }
+        if (this.behavioral) {
+            this.behavioral.destroy();
+        }
+    }
+}
+
+module.exports = DetectionEngine;

package/detection/patterns.js

@@ -0,0 +1,83 @@
+// honeyweb-core/detection/patterns.js
+// SQLi and XSS pattern detection (extracted from main index.js)
+
+// Malicious patterns for SQLi and XSS detection
+const MALICIOUS_PATTERNS = [
+    // SQL Injection patterns
+    /(\bUNION\b.*\bSELECT\b)/i,
+    /(\bOR\b\s+\d+\s*=\s*\d+)/i,
+    /(\bAND\b\s+\d+\s*=\s*\d+)/i,
+    /(';?\s*DROP\s+TABLE)/i,
+    /(';?\s*DELETE\s+FROM)/i,
+    /(\bEXEC\b\s*\()/i,
+    /(\bINSERT\s+INTO\b)/i,
+    /(--\s*$)/,
+    /(';\s*--)/,
+
+    // XSS patterns
+    /(<script[^>]*>.*?<\/script>)/i,
+    /(<iframe[^>]*>)/i,
+    /(<img[^>]*onerror\s*=)/i,
+    /(javascript\s*:)/i,
+    /(<svg[^>]*onload\s*=)/i,
+    /(on\w+\s*=\s*["'][^"']*["'])/i,
+    /(<object[^>]*>)/i,
+    /(<embed[^>]*>)/i
+];
+
+/**
+ * Check if request contains malicious patterns
+ * @param {Object} req - Express request object
+ * @returns {Object} - { detected: boolean, threats: string[] }
+ */
+function detectMaliciousPatterns(req) {
+    const threats = [];
+
+    // Check URL
+    const url = req.url || '';
+    for (const pattern of MALICIOUS_PATTERNS) {
+        if (pattern.test(url)) {
+            threats.push(`Malicious pattern in URL: ${pattern.source.substring(0, 50)}`);
+        }
+    }
+
+    // Check query parameters
+    const query = JSON.stringify(req.query || {});
+    for (const pattern of MALICIOUS_PATTERNS) {
+        if (pattern.test(query)) {
+            threats.push(`Malicious pattern in query: ${pattern.source.substring(0, 50)}`);
+        }
+    }
+
+    // Check body (if exists)
+    if (req.body) {
+        const body = JSON.stringify(req.body);
+        for (const pattern of MALICIOUS_PATTERNS) {
+            if (pattern.test(body)) {
+                threats.push(`Malicious pattern in body: ${pattern.source.substring(0, 50)}`);
+            }
+        }
+    }
+
+    // Check headers (User-Agent, Referer, etc.)
+    const userAgent = req.headers['user-agent'] || '';
+    const referer = req.headers['referer'] || '';
+    for (const pattern of MALICIOUS_PATTERNS) {
+        if (pattern.test(userAgent)) {
+            threats.push(`Malicious pattern in User-Agent`);
+        }
+        if (pattern.test(referer)) {
+            threats.push(`Malicious pattern in Referer`);
+        }
+    }
+
+    return {
+        detected: threats.length > 0,
+        threats
+    };
+}
+
+module.exports = {
+    MALICIOUS_PATTERNS,
+    detectMaliciousPatterns
+};

package/detection/rate-limiter.js

@@ -0,0 +1,109 @@
+// honeyweb-core/detection/rate-limiter.js
+// Rate limiting with auto-cleanup (fixes memory leak)
+
+class RateLimiter {
+    constructor(config) {
+        this.window = config.window || 10000; // 10 seconds
+        this.maxRequests = config.maxRequests || 50;
+        this.requests = new Map(); // ip -> [timestamps]
+
+        // Auto-cleanup to prevent memory leak
+        const cleanupInterval = config.cleanupInterval || 60000; // 60 seconds
+        this.cleanupTimer = setInterval(() => {
+            this._cleanup();
+        }, cleanupInterval);
+    }
+
+    /**
+     * Check if IP has exceeded rate limit
+     * @param {string} ip
+     * @returns {Object} - { limited: boolean, count: number, remaining: number }
+     */
+    check(ip) {
+        const now = Date.now();
+        const windowStart = now - this.window;
+
+        // Get existing requests for this IP
+        let timestamps = this.requests.get(ip) || [];
+
+        // Filter out old requests outside the window
+        timestamps = timestamps.filter(ts => ts > windowStart);
+
+        // Add current request
+        timestamps.push(now);
+
+        // Update map
+        this.requests.set(ip, timestamps);
+
+        const count = timestamps.length;
+        const remaining = Math.max(0, this.maxRequests - count);
+        const limited = count > this.maxRequests;
+
+        return {
+            limited,
+            count,
+            remaining,
+            resetAt: now + this.window
+        };
+    }
+
+    /**
+     * Clean up old entries to prevent memory leak
+     * @private
+     */
+    _cleanup() {
+        const now = Date.now();
+        const windowStart = now - this.window;
+
+        for (const [ip, timestamps] of this.requests.entries()) {
+            // Filter out old timestamps
+            const active = timestamps.filter(ts => ts > windowStart);
+
+            if (active.length === 0) {
+                // No active requests, remove entry
+                this.requests.delete(ip);
+            } else {
+                // Update with filtered timestamps
+                this.requests.set(ip, active);
+            }
+        }
+    }
+
+    /**
+     * Get statistics
+     * @returns {Object}
+     */
+    getStats() {
+        return {
+            trackedIPs: this.requests.size,
+            window: this.window,
+            maxRequests: this.maxRequests
+        };
+    }
+
+    /**
+     * Reset rate limit for an IP
+     * @param {string} ip
+     */
+    reset(ip) {
+        this.requests.delete(ip);
+    }
+
+    /**
+     * Clear all rate limit data
+     */
+    clear() {
+        this.requests.clear();
+    }
+
+    /**
+     * Cleanup and stop timers
+     */
+    destroy() {
+        if (this.cleanupTimer) {
+            clearInterval(this.cleanupTimer);
+        }
+    }
+}
+
+module.exports = RateLimiter;

package/detection/whitelist.js

@@ -0,0 +1,116 @@
+// honeyweb-core/detection/whitelist.js
+// Legitimate bot whitelist with DNS verification
+
+const DNSVerifier = require('../utils/dns-verify');
+
+// Known legitimate bots with their domain patterns
+const KNOWN_BOTS = {
+    'Googlebot': {
+        patterns: [/Googlebot/i],
+        domains: ['googlebot.com', 'google.com']
+    },
+    'Bingbot': {
+        patterns: [/bingbot/i],
+        domains: ['search.msn.com']
+    },
+    'Slackbot': {
+        patterns: [/Slackbot/i, /Slack-ImgProxy/i],
+        domains: ['slack.com']
+    },
+    'facebookexternalhit': {
+        patterns: [/facebookexternalhit/i],
+        domains: ['facebook.com']
+    },
+    'Twitterbot': {
+        patterns: [/Twitterbot/i],
+        domains: ['twitter.com']
+    },
+    'LinkedInBot': {
+        patterns: [/LinkedInBot/i],
+        domains: ['linkedin.com']
+    },
+    'Applebot': {
+        patterns: [/Applebot/i],
+        domains: ['apple.com']
+    },
+    'DuckDuckBot': {
+        patterns: [/DuckDuckBot/i],
+        domains: ['duckduckgo.com']
+    }
+};
+
+class BotWhitelist {
+    constructor(config) {
+        this.enabled = config.enabled !== false;
+        this.verifyDNS = config.verifyDNS !== false;
+        this.dnsVerifier = new DNSVerifier(config.cacheTTL);
+
+        // Allow custom bot list
+        this.customBots = config.bots || [];
+    }
+
+    /**
+     * Check if request is from a legitimate bot
+     * @param {Object} req - Express request object
+     * @param {string} ip - Client IP address
+     * @returns {Promise<Object>} - { isLegitimate: boolean, botName: string, verified: boolean }
+     */
+    async check(req, ip) {
+        if (!this.enabled) {
+            return { isLegitimate: false, botName: null, verified: false };
+        }
+
+        const userAgent = req.headers['user-agent'] || '';
+
+        // Check against known bots
+        for (const [botName, botInfo] of Object.entries(KNOWN_BOTS)) {
+            // Check if User-Agent matches
+            const matches = botInfo.patterns.some(pattern => pattern.test(userAgent));
+
+            if (matches) {
+                // If DNS verification is disabled, trust the User-Agent
+                if (!this.verifyDNS) {
+                    return {
+                        isLegitimate: true,
+                        botName,
+                        verified: false
+                    };
+                }
+
+                // Verify with DNS
+                for (const domain of botInfo.domains) {
+                    const verification = await this.dnsVerifier.verify(ip, domain);
+
+                    if (verification.verified) {
+                        return {
+                            isLegitimate: true,
+                            botName,
+                            verified: true,
+                            hostname: verification.hostname
+                        };
+                    }
+                }
+
+                // User-Agent claims to be bot but DNS verification failed
+                return {
+                    isLegitimate: false,
+                    botName: `Fake ${botName}`,
+                    verified: false,
+                    reason: 'DNS verification failed'
+                };
+            }
+        }
+
+        // Not a known bot
+        return { isLegitimate: false, botName: null, verified: false };
+    }
+
+    /**
+     * Clear DNS cache
+     */
+    clearCache() {
+        this.dnsVerifier.clearCache();
+    }
+}
+
+module.exports = BotWhitelist;