@anonympins/fingerprint 0.0.5 → 0.0.6

package/README.md

@@ -19,6 +19,7 @@ The process unfolds in three steps:
     *   **Device Behavior**: Rapid fingerprint changes (User-Agent rotation).
     *   **IP Behavior**: An excessive number of different devices seen from the same IP, or a single device using a large number of IPs (proxy rotation).
     *   **Inconsistency**: A low similarity score between the current fingerprint and the initial one associated with the `device_id` (cookie theft detection).
+    *   **Request Patterns**: Repetitive, rapid-fire, or sequential requests typical of scraping bots. The parameters for detecting these patterns (e.g., request velocity, burst detection) are dynamically adjusted by the auto-tuner for optimal performance.
     *   **Honeypot Trap**: Detection of bots that automatically fill hidden form fields or probe for common but unused URL parameters (e.g., `?debug=true`).
 3.  **Dynamic Challenge**: If the suspicion score exceeds a certain threshold, a challenge is presented to the user. The difficulty and type of challenge depend on the score:
     *   **Low to Medium Suspicion**: A combined CPU and Memory Proof-of-Work (PoW) challenge is issued. The difficulty of both the CPU (hash calculation) and Memory (allocation and computation) components scales progressively with the suspicion score. For low scores, the memory challenge is negligible, making it primarily a CPU task.
@@ -33,7 +34,7 @@ Once the challenge is solved, a clearance "ticket" is issued via a secure cookie
 -   **Pluggable Datastore**: Supports external datastores like Redis for state persistence and scalability across multiple server instances.
 -   **Express.js Middleware**: Easy integration into an Express application with `powMiddleware`. The datastore must support setting a Time-To-Live (TTL) for challenge secrets.
 -   **Timing Attack Protection**: Uses `crypto.timingSafeEqual` for secure ticket validation.
--   **Automatic Threshold Tuning**: Includes a genetic algorithm-based optimizer (`startThresholdAutoTuning`) that analyzes real traffic to dynamically adjust suspicion thresholds (`low`, `medium`, `high`), improving bot detection accuracy and reducing false positives over time.
+-   **Automatic Parameter Tuning**: Includes a genetic algorithm-based optimizer (`startThresholdAutoTuning`) that analyzes real traffic to dynamically adjust not only suspicion thresholds (`low`, `medium`, `high`) but also the parameters for behavioral pattern detection, improving accuracy and reducing false positives over time.
 
 ## Installation and Usage
 
@@ -66,6 +67,10 @@ app.use(cookieParser());
 app.use(bodyParser.json()); // For parsing application/json
 app.use(bodyParser.urlencoded({ extended: true })); // For parsing application/x-www-form-urlencoded
 
+// Array to store traffic analysis data for the auto-tuner.
+// In a real application, this could be a more robust logging system (e.g., writing to a file or a database).
+const trafficData = [];
+
 // Configuration of weights and thresholds for calculating the suspicion score.
 // These values should be adjusted based on traffic and expected user behavior.
 const securityConfig = {
@@ -73,6 +78,7 @@ const securityConfig = {
         historyScore: 0.3,       // Penalizes IP rotation (proxy)
         rotationScore: 0.5,      // Penalizes rapid fingerprint changes (user-agent, etc.)
         headerAnomalyScore: 0.1, // Penalizes abnormal headers (missing UA, etc.)
+        requestPatternScore: 0.6,// Penalizes bot-like request sequences (scraping, etc.)
         inconsistencyScore: 0.8, // Strongly penalizes inconsistency between the current and initial fingerprint (stolen cookie)
         honeypotScore: 1.0       // Strongly penalizes bots filling hidden form fields
     },
@@ -83,6 +89,13 @@ const securityConfig = {
         block: 95,  // Score above which the request is blocked outright (HTTP 403)
         isStaticResource: (req) => req.path.startsWith('/static/') // Optional: Custom function to identify static resources
     },
+    patterns: { // (Optional) Initial values for request pattern detection, optimized by auto-tuner if enabled.
+        velocityThreshold: 200, // ms between requests to be considered "fast"
+        burstThreshold: 500,    // ms for identical requests to be a "burst"
+        scrapeThreshold: 1000,  // ms for sequential requests to be "scraping"
+        historySize: 10,        // Number of requests to keep for pattern analysis
+        decayFactor: 0.9,       // How quickly the pattern score decays over time
+    },
     honeypot: {
         // List of field names that are traps for bots.
         // These should be hidden in forms for humans, or be URL parameters your app never uses.
@@ -92,7 +105,15 @@ const securityConfig = {
         trapUrls: ['/wp-admin', '/.env', '/admin.php', '/phpmyadmin'], // (Optional)
         // Automatically detect common SQL/NoSQL injection and RCE patterns in request values. (Optional, default: true)
         detectInjections: true
-    }
+    },
+    // The logger is required for auto-tuning. It collects data on requests.
+    logger: (log) => trafficData.push(log),
+    // (Optional) Configuration for the automatic threshold and pattern tuning.
+    autotuning: {
+        trafficData: trafficData,       // The data source for the genetic algorithm.
+        interval: 1800000,              // Optimization cycle every 30 minutes (in ms).
+        minDataPoints: 200              // Minimum requests before starting an optimization cycle.
+    },
 };
 
 // Create an instance of the middleware with your security configuration.
@@ -268,42 +289,6 @@ const server = http.createServer(async (req, res) => {
 server.listen(3000, () => console.log('Server with manual fingerprint engine started on port 3000'));
 ```
 
-### Automatic Threshold Tuning
-
-Manually setting the `low`, `medium`, and `high` thresholds can be challenging. This library provides a powerful tool to automate this process based on real traffic data. It uses a genetic algorithm to find the optimal thresholds that maximize bot detection while minimizing the impact on legitimate users.
-
-#### How to use it:
-
-1.  **Enable Logging**: The auto-tuner needs data. You must provide a `logger` function in your security configuration. This function will be called for significant events (`challenge_issued`, `challenge_solved`, etc.).
-
-2.  **Enable Auto-tuning**: Add an `autotuning` property to your security configuration. The middleware will automatically start the tuning process.
-
-```javascript
-import { powMiddleware } from './fingerprint.js';
-
-// Array to store traffic analysis data. In a real application, this could be
-// a more robust logging system.
-const trafficData = [];
-
-const securityConfig = {
-    weights: { /* ... */ },
-    thresholds: {
-        low: 20,    // Initial values, will be optimized
-        medium: 45,
-        high: 75
-    },
-    logger: (log) => trafficData.push(log), // The logger is required for auto-tuning
-    autotune: {
-        trafficData: trafficData,       // The data source for the algorithm
-        interval: 1800000,              // Optimization cycle every 30 minutes (optional)
-        minDataPoints: 200              // Minimum requests before starting optimization (optional)
-    }
-};
-
-const powMiddlewareInstance = powMiddleware(securityConfig);
-app.use(powMiddlewareInstance);
-```
-
 ---
 
 ## License

package/fingerprint.js

@@ -437,16 +437,20 @@ function getHoneypotScore(context, honeypotConfig = {}) {
   // 3. Check for injection attempts in values
   if (detectInjections) {
     // Regex for common SQL injection patterns
+    // WARNING: These are generic and may cause false positives.
+    // Consider using a dedicated WAF library or more specific regex for your application.
     const sqlRegex = new RegExp(
-      "('|\"|;|--|#|/\\*.*\\*/)|\\b(union|select|insert|update|delete|drop|truncate)\\b",
+      "('|\"|;|--|#|/\\*.*\\*/)|\\b(union|select|insert|update|delete|drop|truncate|from|where|and|or)\\b",
       "i"
     );
     // Regex for common NoSQL (MongoDB) injection patterns (e.g., keys starting with '$')
-    const nosqlKeyRegex = /"\$[^"]*":/;
+    // This looks for keys like "$where", "$ne", etc. in a stringified JSON.
+    const nosqlKeyRegex = /"\$(where|ne|gt|lt|in|nin)":/;
     // Regex for common Remote Code Execution (RCE) patterns
     const rceRegex = new RegExp(
       // File traversal, command execution functions, and shell commands
-      "(\\.\\./|\\.\\.\\\\)|\\b(exec|system|shell_exec|passthru|popen|proc_open|eval|assert|require|include)(_once)?\\s*\\(|\\b(wget|curl|bash|sh|powershell)\\b",
+      // Added process, child_process to catch Node.js specific RCE.
+      "(\\.\\./|\\.\\.\\\\)|\\b(exec|system|shell_exec|passthru|popen|proc_open|eval|assert|require|include|process|child_process)(_once)?\\s*\\(|\\b(wget|curl|bash|sh|powershell|php)\\b",
       "i"
     );
 
@@ -478,6 +482,93 @@ function getHoneypotScore(context, honeypotConfig = {}) {
   return { honeypotScore: 0 };
 }
 
+/**
+ * Analyzes server-side request patterns for a given device to detect bot-like behavior.
+ * This is a stateful check that looks for repetitive or unnaturally fast requests.
+ * @param {object} context - The request context.
+ * @param {object} deviceData - The device's activity data from the store.
+ * @returns {{requestPatternScore: number}}
+ */
+function getRequestPatternScore(context, deviceData, patternConfig = {}) {
+    if (!deviceData) return { requestPatternScore: 0 };
+
+    // Default values for the pattern detection logic, which can be overridden by the auto-tuner.
+    const {
+        velocityThreshold = 200, velocityWeight = 30,
+        burstThreshold = 500, burstWeight = 50,
+        scrapeThreshold = 1000, scrapeWeight = 20, scrapeBurstWeight = 40,
+        historySize = 10,
+        decayFactor = 0.9,
+        inactivityReset = 30000
+    } = patternConfig;
+
+    const now = Date.now();
+    const currentPath = context.path;
+    // Make the function robust to handle both URLSearchParams and plain objects for query.
+    // Ensure query parameters are consistently handled, whether they come from a URLSearchParams object or a plain object.
+    const params = context.query instanceof URLSearchParams ? context.query : new URLSearchParams(context.query);
+    params.sort(); // Sort for deterministic order
+    const currentQueryString = params.toString();
+
+    // Initialize request history if it doesn't exist
+    if (!deviceData.requestHistory) {
+        deviceData.requestHistory = [];
+    }
+
+    const history = deviceData.requestHistory;
+    let score = 0;
+
+    // --- Analyze patterns based on the last few requests ---
+    if (history.length > 0) {
+        const lastRequest = history[history.length - 1];
+        const timeSinceLast = now - lastRequest.timestamp; // 150
+
+        // 1. Velocity Check: Penalize requests that are too fast to be human.
+        if (timeSinceLast < velocityThreshold) { // 150 < 200 -> true
+            score += velocityWeight; // score = 30
+        }
+
+        // 2. Burst Check: Add additional penalty for identical requests in a very short time frame.
+        if (currentPath === lastRequest.path && currentQueryString === lastRequest.queryString && timeSinceLast < burstThreshold) { // 150 < 500 -> true
+            score += burstWeight; // score = 30 + 50 = 80
+        }
+
+        // 3. Sequential Scraping Check: Add additional penalty for same path with different query params (potential scraping).
+        // This is a simplified check, now independent of the burst check.
+        if (currentPath === lastRequest.path && currentQueryString !== lastRequest.queryString && timeSinceLast < scrapeThreshold) {
+            const previousRequest = history.length > 2 ? history[history.length - 2] : null;
+            if (previousRequest && previousRequest.path === currentPath) {
+                score += scrapeBurstWeight; // This is at least the 3rd request in a sequence to the same path.
+            } else {
+                score += scrapeWeight; // First sign of a potential scraping pattern
+            }
+        }
+    }
+
+    // --- Update history ---
+    history.push({
+        timestamp: now,
+        path: currentPath,
+        queryString: currentQueryString,
+    });
+
+    // Keep history to a reasonable size (e.g., last 10 requests)
+    if (history.length > historySize) {
+        history.shift();
+    }
+
+    // Decay the score over time if behavior becomes normal again.
+    // We can store the score in deviceData and decay it.
+    deviceData.lastPatternScore = (deviceData.lastPatternScore || 0) * decayFactor + score; // Decay old score and add new
+
+    // If there hasn't been a request in a while, reset the pattern score.
+    if (history.length > 1 && (now - history[history.length - 2].timestamp > inactivityReset)) { // X ms inactivity
+        deviceData.lastPatternScore = 0;
+    }
+
+    return { requestPatternScore: Math.min(100, deviceData.lastPatternScore) };
+}
+
 const trapUrlTemplates = [
     '/includes/config-{RANDOM}.php',          // Classic PHP config file
     '/.env.{RANDOM}',                         // Environment file
@@ -512,7 +603,13 @@ function generateTrapUrl(nonce) {
  */
 function verifyTrapUrl(path, signature, nonce) {
     const expectedSignature = crypto.createHmac('sha256', getPowSecret()).update(nonce + path).digest('hex').substring(0, 16);
-    return signature === expectedSignature;
+    try {
+        // Use timingSafeEqual to prevent timing attacks where an attacker could guess the signature byte by byte.
+        return crypto.timingSafeEqual(Buffer.from(signature, 'hex'), Buffer.from(expectedSignature, 'hex'));
+    } catch {
+        // This will catch errors if buffers have different lengths or contain invalid hex characters, which is a failure case.
+        return false;
+    }
 }
 /**
  * @typedef {object} IStore
@@ -589,6 +686,7 @@ async function resolveRequestIdentity(context) {
     deviceData = {
       initialDeviceHash: currentDeviceHash, // Anchor the initial fingerprint.
       ips: new Set(),
+      requestHistory: [], // Initialize state for the new pattern score
       lastUpdate: Date.now(),
       lastFpHash: currentDeviceHash,
       lastChangeTimestamp: 0,
@@ -935,27 +1033,45 @@ class FingerprintEngine {
 
   async processRequest(requestContext) {
     const { clientIp, path, cookies, query, isStatic } = requestContext;
-    const { weights, thresholds, logger } = this.securityConfig;
+    const { weights, thresholds, logger, onDeviceCompromised } = this.securityConfig;
 
     if (isStatic) {
       return { action: 'next', score: 0, vector: {} };
     }
 
+    const { pow_nonce } = query;
+
+    // Honeypot: Direct probing of challenge endpoints is highly suspicious.
+    // A legitimate user only hits these endpoints via the challenge page itself, which is only served to suspicious users.
+    // If we see a pow_nonce on a request that isn't (yet) considered suspicious, it's a bot.
+    // We check this early, before the main suspicion calculation.
+    if (pow_nonce) {
+        const powCookie = cookies?.pow_clearance;
+        if (!isTicketValid(clientIp, powCookie)) { // Only check if there's no valid ticket
+            // This is a potential probe. We'll let the main logic confirm if it's not a legitimate challenge response.
+        }
+    }
+
     // Check for persisted "condemned" status early.
     const { deviceData } = await resolveRequestIdentity(requestContext);
     if (deviceData?.condemned) {
+        if (onDeviceCompromised) {
+            onDeviceCompromised({ deviceId: cookies?.device_id, clientIp, reason: 'Previously condemned', score: 100, vector: { honeypotScore: 100 } });
+        }
         return { action: 'block', status: 403, body: 'Forbidden', score: 100, vector: { honeypotScore: 100 } };
     }
 
     // The engine now works with the context directly, no more rawReq dependency here.
     const suspicionVector = await __internal.getSuspicionVector(requestContext, this.securityConfig);
     const { honeypotScore } = getHoneypotScore(requestContext, this.securityConfig.honeypot);
+    const { requestPatternScore } = getRequestPatternScore(requestContext, deviceData, this.securityConfig.patterns);
     suspicionVector.honeypotScore = honeypotScore;
+    suspicionVector.requestPatternScore = requestPatternScore;
 
     const finalScore =
       suspicionVector.historyScore * (weights.historyScore || 0) +
       suspicionVector.rotationScore * (weights.rotationScore || 0) +
-      suspicionVector.headerAnomalyScore * (weights.headerAnomalyScore || 0) +
+      suspicionVector.headerAnomalyScore * (weights.headerAnomalyScore || 0) + suspicionVector.requestPatternScore * (weights.requestPatternScore || 0) +
       suspicionVector.inconsistencyScore * (weights.inconsistencyScore || 0) +
       honeypotScore * (weights.honeypotScore || 0);
 
@@ -973,7 +1089,7 @@ class FingerprintEngine {
         )
         : 0;
     const powCookie = cookies?.pow_clearance;
-    const { pow_type, pow_nonce, pow_solution, pow_solution_cpu, pow_solution_mem } = query;
+    const { pow_type, pow_solution, pow_solution_cpu, pow_solution_mem } = query;
 
     // Honeypot: Direct probing of challenge endpoints is highly suspicious.
     // A legitimate user only hits these endpoints via the challenge page itself.
@@ -983,12 +1099,15 @@ class FingerprintEngine {
       }
       suspicionVector.honeypotScore = 100; // Bot is probing. Max penalty.
       // Recalculate score and block immediately.
-      const newFinalScore = finalScore + (100 * (weights.honeypotScore || 0));
+      const newFinalScore = finalScore - (honeypotScore * (weights.honeypotScore || 0)) + (100 * (weights.honeypotScore || 0));
       return { action: 'block', status: 403, body: 'Forbidden', score: newFinalScore, vector: suspicionVector };
     }
 
     // If the action is to block, we should still include the score and vector for logging/testing.
     if (isBlocked) {
+      if (onDeviceCompromised) {
+        onDeviceCompromised({ deviceId: cookies?.device_id, clientIp, reason: 'Score exceeded block threshold', score: finalScore, vector: suspicionVector });
+      }
       return { action: 'block', status: 403, body: 'Forbidden', score: finalScore, vector: suspicionVector };
     }
 
@@ -997,6 +1116,9 @@ class FingerprintEngine {
     const lastNonce = deviceData?.lastChallengeNonce;
     if (lastNonce && query.sig && verifyTrapUrl(path, query.sig, lastNonce)) {
         deviceData.condemned = true; // This device is a bot. Condemn it.
+        if (onDeviceCompromised) {
+            onDeviceCompromised({ deviceId: cookies?.device_id, clientIp, reason: 'Triggered signed honeypot trap URL', score: 100, vector: { honeypotScore: 100 } });
+        }
         await store.set(`device:${cookies.device_id}`, deviceData);
         return { action: 'block', status: 403, body: 'Forbidden', score: 100, vector: { honeypotScore: 100 } };
     }
@@ -1163,12 +1285,14 @@ class FingerprintEngine {
 
     const vector = await __internal.getSuspicionVector(requestContext, this.securityConfig); // Pass the config
     const { honeypotScore } = getHoneypotScore(requestContext, this.securityConfig.honeypot);
+    const { requestPatternScore } = getRequestPatternScore(requestContext, (await store.get(`device:${requestContext.cookies?.device_id}`)), this.securityConfig.patterns);
     const score =
       vector.historyScore * (this.securityConfig.weights.historyScore || 0.3) +
       vector.rotationScore * (this.securityConfig.weights.rotationScore || 0.5) +
       vector.headerAnomalyScore * (this.securityConfig.weights.headerAnomalyScore || 0.1) +
       vector.inconsistencyScore * (this.securityConfig.weights.inconsistencyScore || 0.8) +
-      honeypotScore * (this.securityConfig.weights.honeypotScore || 0);
+      honeypotScore * (this.securityConfig.weights.honeypotScore || 0) +      
+      requestPatternScore * (this.securityConfig.weights.requestPatternScore || 0);
 
     if (score >= this.securityConfig.thresholds.high) return `suspicious_high:${clientIp}`;
     if (score >= this.securityConfig.thresholds.low) return `suspicious_medium:${clientIp}`; // Use medium for any suspicion
@@ -1252,6 +1376,7 @@ export const __internal = {
     FingerprintBuilder, // Export for testing
     calculateTarget,
     FingerprintEngine, // Expose for advanced testing
+    getRequestPatternScore, // Expose for testing
 };
 
 // --- THRESHOLD AUTO-TUNING SECTION ---
@@ -1286,9 +1411,11 @@ function runThresholdOptimization(securityConfig, trafficData, minDataPoints) {
     // The "fitness" function evaluates the quality of a set of thresholds.
     // A lower score is better.
     const fitnessFunction = (solution) => {
-        const [low, medium, high] = solution;
+        const [low, medium, high, velocityThreshold, burstThreshold, scrapeThreshold] = solution;
         // Constraints: thresholds must be ordered and within a reasonable range.
-        if (low >= medium || medium >= high || low <= 10 || high >= 90) return Infinity;
+        if (low >= medium || medium >= high || low < 10 || high > 90) return Infinity;
+        // Constraints for pattern thresholds
+        if (velocityThreshold < 50 || velocityThreshold > burstThreshold || burstThreshold > scrapeThreshold) return Infinity;
 
         let falsePositives = 0; // Humans challenged unnecessarily.
         let falseNegatives = 0; // Undetected bots.
@@ -1305,12 +1432,21 @@ function runThresholdOptimization(securityConfig, trafficData, minDataPoints) {
     };
 
     // Functions for the genetic algorithm.
-    const createIndividual = () => [10 + Math.random() * 20, 30 + Math.random() * 30, 60 + Math.random() * 30];
-    const crossover = (p1, p2) => [(p1[0] + p2[0]) / 2, (p1[1] + p2[1]) / 2, (p1[2] + p2[2]) / 2];
+    const createIndividual = () => [
+        10 + Math.random() * 20, // low
+        30 + Math.random() * 30, // medium
+        60 + Math.random() * 30, // high
+        100 + Math.random() * 150, // velocityThreshold (100-250ms)
+        300 + Math.random() * 400, // burstThreshold (300-700ms)
+        800 + Math.random() * 700, // scrapeThreshold (800-1500ms)
+    ];
+    const crossover = (p1, p2) => p1.map((val, i) => (val + p2[i]) / 2);
     const mutate = (s) => {
         const n = [...s];
-        const i = Math.floor(Math.random() * 3);
-        n[i] += (Math.random() - 0.5) * 5;
+        const i = Math.floor(Math.random() * n.length);
+        // Adjust mutation range based on parameter
+        const mutationRange = i < 3 ? 5 : 50;
+        n[i] += (Math.random() - 0.5) * mutationRange;
         return n;
     };
 
@@ -1320,16 +1456,26 @@ function runThresholdOptimization(securityConfig, trafficData, minDataPoints) {
         populationSize: 40
     });
 
-    const [newLow, newMedium, newHigh] = result.solution;
+    const [newLow, newMedium, newHigh, newVelocity, newBurst, newScrape] = result.solution;
 
     // Update the configuration live.
-    securityConfig.thresholds = {
-        low: Math.round(newLow),
-        medium: Math.round(newMedium),
-        high: Math.round(newHigh)
-    };
+    // Ensure thresholds object exists
+    if (!securityConfig.thresholds) securityConfig.thresholds = {};
+    securityConfig.thresholds.low = Math.round(newLow);
+    securityConfig.thresholds.medium = Math.round(newMedium);
+    securityConfig.thresholds.high = Math.round(newHigh);
+
+    // Update pattern detection parameters
+    if (!securityConfig.patterns) securityConfig.patterns = {};
+    securityConfig.patterns.velocityThreshold = Math.round(newVelocity);
+    securityConfig.patterns.burstThreshold = Math.round(newBurst);
+    securityConfig.patterns.scrapeThreshold = Math.round(newScrape);
+    // Weights could also be optimized, but let's keep it to thresholds for now for simplicity.
 
     console.log("[AutoTuning] Nouveaux seuils optimisés appliqués :", securityConfig.thresholds);
+    if (securityConfig.patterns) {
+        console.log("[AutoTuning] Nouveaux paramètres de pattern appliqués :", securityConfig.patterns);
+    }
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anonympins/fingerprint",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "description": "An HTTP(S) client mitigation and anti-bot protection library for Node.js/Express, based on digital fingerprinting and dynamic Proof-of-Work (PoW) challenges.",
   "main": "fingerprint.js",
   "type": "module",