@hadi_ali/warden 1.0.0

package/app/scoring/headerFingerPrint.js

@@ -0,0 +1,16 @@
+const crypto = require("crypto");
+
+function headerFingerPrint(req) {
+  const headers = req.headers || {};
+  const entries = Object.keys(headers)
+    .sort()
+    .map(k => `${k}:${headers[k]}`)
+    .join(',');
+  return crypto
+    .createHash("sha256")
+    .update(entries)
+    .digest("hex")
+    .slice(0, 16);
+}
+
+module.exports = { headerFingerPrint };

package/app/scoring/scoreFromHeaders.js

@@ -0,0 +1,37 @@
+const { scoreAgent } = require('./scoreUser');
+
+function scoreFromHeaders(req) {
+  let score = 0;
+  const ua = req.headers['user-agent'] || '';
+  const acceptEncoding = req.headers['accept-encoding'] || '';
+
+  const uaScore = scoreAgent(ua, req.headers);
+  const fingerPrintScore = Object.keys(req.headers).length === 0 ? 10 : 0;
+  
+
+  const isChromium = /Chrome\/([9][0-9]|[1-9][0-9]{2})/i.test(ua); // Chrome 90+
+  if (isChromium) {
+    if (!req.headers['sec-ch-ua']) {
+      score += 40;
+    }
+    if (!req.headers['sec-fetch-site']) {
+      score += 20;
+    }
+    const hasBrotli = acceptEncoding.includes('br');
+    const hasGzip = acceptEncoding.includes('gzip');
+    if (!hasBrotli && !hasGzip) {
+      score += 10;
+    }
+  }
+
+  const secPlatform = req.headers['sec-ch-ua-platform'];
+  if (secPlatform) {
+    if (ua.includes('Windows') && !secPlatform.includes('Windows')) score += 50;
+    if (ua.includes('Macintosh') && !secPlatform.includes('macOS')) score += 50;
+    if (ua.includes('Linux') && !secPlatform.includes('Linux')) score += 50;
+  }
+
+  return uaScore + fingerPrintScore + score;
+}
+
+module.exports = { scoreFromHeaders };

package/app/scoring/scoreUser.js

@@ -0,0 +1,44 @@
+function scoreAgent(UserAgent, headers = {}) {
+  const botPatterns = [
+    /bot/i, /crawl/i, /spider/i, /scan/i,
+    /fetch/i, /slurp/i, /wget/i, /curl/i,
+    /python/i, /java/i, /httpclient/i, /axios/i,
+    /scrapy/i, /puppeteer/i, /selenium/i, /playwright/i
+  ];
+
+  let score = 0;
+
+  if (!headers['accept']) score += 30;
+  if (!headers['accept-language']) score += 20;
+  if (!headers['accept-encoding']) score += 20;
+
+  if (!UserAgent || UserAgent.length === 0) {
+    score += 40;
+    return Math.min(score, 100);
+  }
+  if (UserAgent.length < 20) score += 25;
+  if (UserAgent.length > 200) score += 20;
+
+  for (const pattern of botPatterns) {
+    if (pattern.test(UserAgent)) {
+      if (/selenium|puppeteer|playwright/i.test(UserAgent)) score += 30;
+      else if (/scrapy|crawler/i.test(UserAgent)) score += 25;
+      else score += 15;
+      return Math.min(score, 100);
+    }
+  }
+
+  if (/Mozilla\/5\.0/.test(UserAgent)) {
+    if (/Chrome\/\d+|Firefox\/\d+|Safari\/\d+|Edge\/\d+/.test(UserAgent)) {
+      return Math.min(score, 100);
+    }
+    score += 5;
+  }
+
+  if (/^[^a-zA-Z0-9]/.test(UserAgent)) score += 35;
+  if (/[\x00-\x1F]/.test(UserAgent)) score += 50;
+
+  return Math.min(score, 100);
+}
+
+module.exports = { scoreAgent };

package/app/store.js

@@ -0,0 +1,193 @@
+class Store {
+  constructor({ maxSize = 100000, sweepIntervalMs = 5 * 60 * 1000 } = {}) {
+    this.memorystore = new Map();
+    this.redisstore = null;
+    this.useredis = false;
+    this.maxMapSize = maxSize;
+    this.sweepInterval = null;
+
+    this._startSweep(sweepIntervalMs);
+  }
+
+  _startSweep(intervalMs) {
+    if (this.sweepInterval) return;
+    this.sweepInterval = setInterval(() => this._sweepExpired(), intervalMs);
+    if (this.sweepInterval.unref) this.sweepInterval.unref();
+  }
+
+  _stopSweep() {
+    if (this.sweepInterval) {
+      clearInterval(this.sweepInterval);
+      this.sweepInterval = null;
+    }
+  }
+
+  // Walk the map and delete every entry whose expiresAt < now.
+  // This is the proactive cleanup that prevents the leak the old
+  // "delete on next access" design had.
+  _sweepExpired() {
+    if (this.useredis) return;
+    const now = Date.now();
+    let removed = 0;
+    for (const [key, entry] of this.memorystore) {
+      if (now > entry.expiresAt) {
+        this.memorystore.delete(key);
+        removed++;
+      }
+    }
+    return removed;
+  }
+
+  // LRU touch: re-insert a key to move it to the end of iteration order.
+  // The oldest (least-recently-used) key is `this.memorystore.keys().next().value`.
+  _touch(key) {
+    if (!this.memorystore.has(key)) return;
+    const entry = this.memorystore.get(key);
+    this.memorystore.delete(key);
+    this.memorystore.set(key, entry);
+  }
+
+  // Evict the least-recently-used entry. Called when at capacity.
+  _evictLRU() {
+    const oldestKey = this.memorystore.keys().next().value;
+    if (oldestKey !== undefined) this.memorystore.delete(oldestKey);
+  }
+
+  async initRedis(url) {
+    if (!url) return;
+    try {
+      const { createClient } = require('redis');
+      this.redisstore = createClient({ url });
+      this.redisstore.on('error', (err) => {
+        console.error('[Redis Error]', err.message);
+        this.useredis = false;
+      });
+      this.redisstore.on('ready', () => {
+        this.useredis = true;
+      });
+      await this.redisstore.connect();
+    } catch (err) {
+      console.error('[Redis Init Error]', err.message);
+      this.useredis = false;
+    }
+  }
+
+  useRedis(client) {
+    if (!client || typeof client.get !== 'function' || typeof client.set !== 'function') {
+      this.useredis = false;
+      this.redisstore = null;
+      return false;
+    }
+    this.redisstore = client;
+    this.useredis = true;
+    return true;
+  }
+
+  async set(key, value, ttlSeconds = 60) {
+    if (this.useredis) {
+      try {
+        const payload = JSON.stringify(value);
+        const client = this.redisstore;
+        try {
+          return await client.set(key, payload, { EX: ttlSeconds });
+        } catch (e1) {
+          try {
+            return await client.setEx(key, ttlSeconds, payload);
+          } catch (e2) {
+            await client.set(key, payload);
+            await client.expire(key, ttlSeconds);
+            return true;
+          }
+        }
+      } catch (err) {
+        return null;
+      }
+    }
+
+    if (this.memorystore.has(key)) this.memorystore.delete(key);
+    if (this.memorystore.size >= this.maxMapSize) this._evictLRU();
+    this.memorystore.set(key, {
+      value,
+      expiresAt: Date.now() + (ttlSeconds * 1000)
+    });
+    return true;
+  }
+
+  async get(key) {
+    if (this.useredis) {
+      try {
+        const raw = await this.redisstore.get(key);
+        if (!raw) return null;
+        return JSON.parse(raw);
+      } catch (err) {
+        return null;
+      }
+    }
+
+    const entry = this.memorystore.get(key);
+    if (!entry) return null;
+
+    if (Date.now() > entry.expiresAt) {
+      this.memorystore.delete(key);
+      return null;
+    }
+
+    // Touch to mark this key as recently used
+    this._touch(key);
+    return entry.value;
+  }
+
+  async delete(key) {
+    if (this.useredis) {
+      try {
+        return await this.redisstore.del(key);
+      } catch (err) {
+        return null;
+      }
+    }
+    return this.memorystore.delete(key);
+  }
+
+  async increment(key, ttlSeconds = 5) {
+    if (this.useredis) {
+      try {
+        const multi = this.redisstore.multi();
+        multi.incr(key);
+        multi.expire(key, ttlSeconds);
+        const results = await multi.exec();
+        return results[0];
+      } catch (err) {
+        return 1;
+      }
+    }
+
+    const now = Date.now();
+    const existing = this.memorystore.get(key);
+
+    if (!existing || now > existing.expiresAt) {
+      if (this.memorystore.has(key)) this.memorystore.delete(key);
+      if (this.memorystore.size >= this.maxMapSize) this._evictLRU();
+      this.memorystore.set(key, { value: 1, expiresAt: now + (ttlSeconds * 1000) });
+      return 1;
+    }
+
+    existing.value += 1;
+    this._touch(key);
+    return existing.value;
+  }
+
+  stats() {
+    return {
+      backend: this.useredis ? 'redis' : 'memory',
+      size: this.useredis ? null : this.memorystore.size,
+      maxSize: this.maxMapSize,
+    };
+  }
+
+  shutdown() {
+    this._stopSweep();
+  }
+}
+
+const store = new Store();
+module.exports = { store, Store };

package/app/verified-bots/matcher.js

@@ -0,0 +1,102 @@
+const fs = require('fs');
+const path = require('path');
+const net = require('net');
+const os = require('os');
+const { Address6 } = require('ip-address');
+
+const RANGES_FILE = path.join(__dirname, 'ranges.json');
+const TMP_RANGES_FILE = path.join(os.tmpdir(), 'warden-ranges.json');
+
+let cachedRanges = { ipv4: [], ipv6: [] };
+let cacheSource = 'empty';
+
+// Load subnets from cache (in-memory, tmpdir file, or bundled file)
+function loadRanges() {
+  try {
+    // 1. In-memory cache (synced from sync.js)
+    const { getMemoryCache } = require('./sync');
+    const fromMemory = getMemoryCache && getMemoryCache();
+    if (fromMemory && (fromMemory.ipv4?.length || fromMemory.ipv6?.length)) {
+      applyRanges(fromMemory);
+      cacheSource = 'memory';
+      return;
+    }
+
+    // 2. Filesystem (prefer tmpdir, fall back to bundled file)
+    const candidates = [TMP_RANGES_FILE, RANGES_FILE];
+    for (const file of candidates) {
+      if (fs.existsSync(file)) {
+        const raw = fs.readFileSync(file, 'utf8');
+        const data = JSON.parse(raw);
+        applyRanges(data);
+        cacheSource = `file:${file}`;
+        return;
+      }
+    }
+
+    console.warn('[warden] No verified bot ranges loaded. Run syncVerifiedBots() to populate.');
+  } catch (err) {
+    console.error('[warden] Failed to load verified bot ranges:', err);
+  }
+}
+
+function applyRanges(data) {
+  cachedRanges.ipv4 = (data.ipv4 || []).map(cidr => {
+    const [subnet, bits] = cidr.split('/');
+    const bitCount = parseInt(bits, 10);
+    const mask = (bitCount === 0) ? 0 : (~(2 ** (32 - bitCount) - 1)) >>> 0;
+    return { subnetInt: ip4ToInt(subnet), mask };
+  });
+
+  cachedRanges.ipv6 = data.ipv6 || [];
+}
+
+function ip4ToInt(ip) {
+  return ip.split('.').reduce((int, oct) => (int << 8) + parseInt(oct, 10), 0) >>> 0;
+}
+
+function matchIPv4(ip) {
+  const ipInt = ip4ToInt(ip);
+  for (let i = 0; i < cachedRanges.ipv4.length; i++) {
+    const { subnetInt, mask } = cachedRanges.ipv4[i];
+    if ((ipInt & mask) === (subnetInt & mask)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function matchIPv6(ip) {
+  try {
+    const clientAddr = new Address6(ip);
+    for (let i = 0; i < cachedRanges.ipv6.length; i++) {
+      const rangeAddr = new Address6(cachedRanges.ipv6[i]);
+      if (clientAddr.isInSubnet(rangeAddr)) {
+        return true;
+      }
+    }
+  } catch (err) {
+    return false;
+  }
+  return false;
+}
+
+function isVerifiedBot(ip) {
+  if (!ip) return false;
+  const version = net.isIP(ip);
+  if (version === 4) return matchIPv4(ip);
+  if (version === 6) return matchIPv6(ip);
+  return false;
+}
+
+function getCacheInfo() {
+  return {
+    source: cacheSource,
+    ipv4_count: cachedRanges.ipv4.length,
+    ipv6_count: cachedRanges.ipv6.length,
+  };
+}
+
+loadRanges();
+
+module.exports = { isVerifiedBot, reloadRanges: loadRanges, getCacheInfo };

package/app/verified-bots/sync.js

@@ -0,0 +1,220 @@
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const net = require('net');
+
+const RANGES_FILE = path.join(__dirname, 'ranges.json');
+const TMP_RANGES_FILE = path.join(os.tmpdir(), 'warden-ranges.json');
+
+const SOURCES = {
+  google: 'https://developers.google.com/search/apis/ipranges/googlebot.json',
+  bing: 'https://www.bing.com/toolbox/bingbot.json'
+};
+
+const FETCH_TIMEOUT_MS = 10000;
+
+// In-memory cache (always populated; this is the source of truth)
+let memoryCache = null;
+
+// Optional external storage (Redis client or DB with set/get)
+let externalStore = null;
+
+async function fetchJSON(url) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+  try {
+    const res = await fetch(url, {
+      headers: { 'User-Agent': 'Warden-Sync-Engine/1.0' },
+      signal: controller.signal
+    });
+    if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`);
+    return await res.json();
+  } catch (err) {
+    if (err && err.name === 'AbortError') {
+      throw new Error(`Timeout after ${FETCH_TIMEOUT_MS}ms fetching ${url}`);
+    }
+    throw err;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function isValidCIDR(cidr) {
+  if (typeof cidr !== 'string') return false;
+  const [ip, bits] = cidr.split('/');
+  if (!ip || !bits) return false;
+
+  const ipVersion = net.isIP(ip);
+  const bitNum = parseInt(bits, 10);
+
+  if (ipVersion === 4) return bitNum >= 0 && bitNum <= 32;
+  if (ipVersion === 6) return bitNum >= 0 && bitNum <= 128;
+  return false;
+}
+
+async function persistToExternal(payload) {
+  if (!externalStore) return;
+  try {
+    if (typeof externalStore.set === 'function') {
+      const json = JSON.stringify(payload);
+      try {
+        await externalStore.set('warden:verified_ranges', json, { EX: 86400 });
+      } catch (e1) {
+        try {
+          await externalStore.setEx('warden:verified_ranges', 86400, json);
+        } catch (e2) {
+          await externalStore.set('warden:verified_ranges', json);
+        }
+      }
+    } else if (typeof externalStore.query === 'function') {
+      await externalStore.query(
+        `INSERT INTO warden_verified_ranges (key, payload, synced_at)
+         VALUES ($1, $2, NOW())
+         ON CONFLICT (key) DO UPDATE SET payload = EXCLUDED.payload, synced_at = NOW()`,
+        ['verified_ranges', JSON.stringify(payload)]
+      );
+    }
+  } catch (err) {
+    console.warn('[warden] Failed to persist ranges to external store:', err.message);
+  }
+}
+
+async function loadFromExternal() {
+  if (!externalStore) return null;
+  try {
+    if (typeof externalStore.get === 'function') {
+      const raw = await externalStore.get('warden:verified_ranges');
+      return raw ? JSON.parse(raw) : null;
+    } else if (typeof externalStore.query === 'function') {
+      const result = await externalStore.query(
+        `SELECT payload FROM warden_verified_ranges WHERE key = $1 LIMIT 1`,
+        ['verified_ranges']
+      );
+      const row = result.rows && result.rows[0];
+      return row ? JSON.parse(row.payload) : null;
+    }
+  } catch (err) {
+    console.warn('[warden] Failed to load ranges from external store:', err.message);
+    return null;
+  }
+  return null;
+}
+
+function setExternalStore(store) {
+  externalStore = store || null;
+}
+
+function setMemoryCache(payload) {
+  memoryCache = payload;
+}
+
+function getMemoryCache() {
+  return memoryCache;
+}
+
+async function syncVerifiedBots({ writeFile = false } = {}) {
+  const ipv4 = new Set();
+  const ipv6 = new Set();
+
+  console.log('[warden] Starting verified bot IP sync...');
+
+  // 1. Sync Googlebot
+  try {
+    const data = await fetchJSON(SOURCES.google);
+    if (Array.isArray(data.prefixes)) {
+      for (const entry of data.prefixes) {
+        const cidr = entry.ipv4Prefix || entry.ipv6Prefix;
+        if (cidr && isValidCIDR(cidr)) {
+          if (entry.ipv4Prefix) ipv4.add(cidr);
+          else ipv6.add(cidr);
+        }
+      }
+    }
+  } catch (err) {
+    console.error('[warden] Error syncing Googlebot ranges:', err.message);
+  }
+
+  // 2. Sync Bingbot
+  try {
+    const data = await fetchJSON(SOURCES.bing);
+    if (Array.isArray(data.prefixes)) {
+      for (const entry of data.prefixes) {
+        const cidr = entry.ipv4Prefix || entry.ipv6Prefix;
+        if (cidr && isValidCIDR(cidr)) {
+          if (entry.ipv4Prefix) ipv4.add(cidr);
+          else ipv6.add(cidr);
+        }
+      }
+    }
+  } catch (err) {
+    console.error('[warden] Error syncing Bingbot ranges:', err.message);
+  }
+
+  // Fail-safe: Do not overwrite with empty arrays if APIs both failed
+  if (ipv4.size === 0 && ipv6.size === 0) {
+    console.error('[warden] Sync failed: No valid ranges retrieved. Keeping existing cache.');
+    return false;
+  }
+
+  const payload = {
+    synced_at: new Date().toISOString(),
+    ipv4: Array.from(ipv4),
+    ipv6: Array.from(ipv6)
+  };
+
+  // Always populate the in-memory cache
+  memoryCache = payload;
+
+  // Try to persist to external store (Redis/DB) if provided
+  await persistToExternal(payload);
+
+  // Optional: write to filesystem (off by default — serverless/containers have read-only FS)
+  if (writeFile) {
+    const targetFile = process.env.WARDEN_RANGES_FILE || TMP_RANGES_FILE;
+    try {
+      fs.writeFileSync(targetFile, JSON.stringify(payload, null, 2));
+      console.log(`[warden] Sync complete. Cached ${ipv4.size} IPv4 and ${ipv6.size} IPv6 ranges at ${targetFile}.`);
+    } catch (err) {
+      console.warn(`[warden] Could not write ranges to ${targetFile}: ${err.message}`);
+    }
+  } else {
+    console.log(`[warden] Sync complete. Loaded ${ipv4.size} IPv4 and ${ipv6.size} IPv6 ranges into memory.`);
+  }
+
+  return true;
+}
+
+async function loadInitialRanges({ writeFile = false } = {}) {
+  // Priority: external store > filesystem > empty
+  const fromExternal = await loadFromExternal();
+  if (fromExternal) {
+    memoryCache = fromExternal;
+    console.log('[warden] Loaded verified bot ranges from external store.');
+    return true;
+  }
+
+  // Try filesystem (prefer tmpdir, fall back to bundled file)
+  for (const file of [TMP_RANGES_FILE, RANGES_FILE]) {
+    try {
+      if (fs.existsSync(file)) {
+        const raw = fs.readFileSync(file, 'utf8');
+        memoryCache = JSON.parse(raw);
+        console.log(`[warden] Loaded verified bot ranges from ${file}.`);
+        return true;
+      }
+    } catch (err) {
+      // try next location
+    }
+  }
+
+  return false;
+}
+
+module.exports = {
+  syncVerifiedBots,
+  loadInitialRanges,
+  setExternalStore,
+  setMemoryCache,
+  getMemoryCache,
+  SOURCES,
+};

package/index.js

@@ -0,0 +1,14 @@
+const { warden } = require('./app/middleware/warden');
+const { migrate } = require('./app/db_connection/migrate');
+const { syncVerifiedBots, SOURCES } = require('./app/verified-bots/sync');
+const { reloadRanges } = require('./app/verified-bots/matcher');
+const { store } = require('./app/store');
+
+module.exports = {
+  warden,
+  migrate,
+  syncVerifiedBots,
+  reloadRanges,
+  SOURCES,
+  store,
+};

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@hadi_ali/warden",
+  "version": "1.0.0",
+  "description": "Express middleware that scores requests and blocks bots/scrapers using header heuristics, IP reputation, rate limiting, and brute-force tracking.",
+  "main": "index.js",
+  "scripts": {
+    "test": "jest",
+    "nodemon": "nodemon",
+    "migrate": "node ./app/db_connection/migrate.js"
+  },
+  "keywords": [
+    "middleware",
+    "express",
+    "security",
+    "antibot",
+    "antispam",
+    "scoring",
+    "rate-limit",
+    "brute-force",
+    "bot-detection",
+    "waf"
+  ],
+  "author": "hadiali",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/hadiali/warden.git"
+  },
+  "bugs": {
+    "url": "https://github.com/hadiali/warden/issues"
+  },
+  "homepage": "https://github.com/hadiali/warden#readme",
+  "engines": {
+    "node": ">=18"
+  },
+  "bin": {
+    "warden": "app/cli/index.js"
+  },
+  "peerDependencies": {
+    "express": "^4.0.0 || ^5.0.0"
+  },
+  "dependencies": {
+    "ip-address": "^10.2.0"
+  },
+  "optionalDependencies": {
+    "pg": "^8.21.0",
+    "redis": "^6.0.0"
+  },
+  "devDependencies": {
+    "dotenv": "^17.4.2",
+    "express": "^5.2.1",
+    "jest": "^30.4.2",
+    "nodemon": "^3.1.14",
+    "source-map-support": "^0.5.21"
+  }
+}