@chdlabs/my-shield 0.1.0

package/src/intelligence/autoPromoter.js

@@ -0,0 +1,91 @@
+/**
+ * Auto-Promoter Engine
+ * Evaluates 404 intelligence data and automatically promotes frequently attacked paths
+ * to active honeypot traps, closing the self-improvement loop.
+ *
+ * @author ChdLabs
+ * @license MIT
+ */
+
+export class AutoPromoter {
+    /**
+     * @param {Object} options
+     * @param {boolean} [options.enabled=true]
+     * @param {number} [options.minHits=50] Minimum 404 hits before promotion
+     * @param {number} [options.minUniqueIPs=10] Minimum distinct IP addresses probing this path
+     * @param {Object} options.collector IntelligenceCollector instance
+     * @param {Object} options.honeypot Honeypot instance
+     * @param {Object} [options.logger=console]
+     */
+    constructor({
+        enabled = true,
+        minHits = 50,
+        minUniqueIPs = 10,
+        collector,
+        honeypot,
+        logger = console
+    } = {}) {
+        this.enabled = enabled;
+        this.minHits = minHits;
+        this.minUniqueIPs = minUniqueIPs;
+        this.collector = collector;
+        this.honeypot = honeypot;
+        this.logger = logger;
+        this.promotedHistory = new Set();
+    }
+
+    /**
+     * Scan current intelligence paths and promote qualifiers to honeypot
+     * @returns {string[]} List of newly promoted path strings
+     */
+    evaluate() {
+        if (!this.enabled || !this.collector || !this.honeypot) {
+            return [];
+        }
+
+        const newlyPromoted = [];
+
+        for (const [path, entry] of this.collector.paths.entries()) {
+            if (entry.autoPromoted || this.promotedHistory.has(path)) {
+                continue;
+            }
+
+            const hitCount = entry.count || 0;
+            const uniqueIpCount = entry.uniqueIPs ? entry.uniqueIPs.size : 0;
+
+            if (hitCount >= this.minHits && uniqueIpCount >= this.minUniqueIPs) {
+                entry.autoPromoted = true;
+                this.promotedHistory.add(path);
+                this.honeypot.addRoute(path, 'auto_promoted');
+                newlyPromoted.push(path);
+
+                if (this.logger && typeof this.logger.info === 'function') {
+                    this.logger.info(`[SHIELD] Auto-promoted path "${path}" to honeypot (${hitCount} hits from ${uniqueIpCount} IPs)`);
+                }
+            }
+        }
+
+        return newlyPromoted;
+    }
+
+    /**
+     * Force-promote a specific path manually
+     * @param {string} path
+     */
+    promotePath(path) {
+        if (!path || typeof path !== 'string') return false;
+        const normalized = this.collector ? this.collector.normalizePath(path) : path;
+        this.promotedHistory.add(normalized);
+        if (this.collector && this.collector.paths.has(normalized)) {
+            this.collector.paths.get(normalized).autoPromoted = true;
+        }
+        if (this.honeypot) {
+            this.honeypot.addRoute(normalized, 'manual_promotion');
+        }
+        return true;
+    }
+
+    getPromotedPaths() {
+        return Array.from(this.promotedHistory);
+    }
+}

package/src/intelligence/collector.js

@@ -0,0 +1,219 @@
+/**
+ * 404 Intelligence Collector
+ * Deduplicates and aggregates 404 path occurrences, tracks scanner user-agents,
+ * and maintains statistical insights on probing patterns.
+ *
+ * @author ChdLabs
+ * @license MIT
+ */
+
+const DEFAULT_EXCLUDE_PATTERNS = [
+    /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff2?|ttf|eot|map)$/i,
+    /^\/favicon\.ico$/i,
+    /^\/robots\.txt$/i,
+    /^\/sitemap\.xml$/i,
+    /^\/apple-touch-icon/i,
+    /^\/manifest\.json$/i
+];
+
+export class IntelligenceCollector {
+    /**
+     * @param {Object} options
+     * @param {number} [options.maxUniquePaths=50000] Maximum unique paths to store before pruning
+     * @param {Array<RegExp>} [options.excludePatterns] Patterns to ignore (e.g. static assets)
+     */
+    constructor({ maxUniquePaths = 50_000, excludePatterns = [] } = {}) {
+        this.maxUniquePaths = Math.max(100, maxUniquePaths);
+        this.excludePatterns = [...DEFAULT_EXCLUDE_PATTERNS, ...excludePatterns];
+        this.paths = new Map(); // path -> { count, firstSeen, lastSeen, uniqueIPs: Set<string>, autoPromoted: boolean }
+        this.scannerFingerprints = new Map(); // userAgent -> count
+        this.startedAt = new Date().toISOString();
+        this.totalHits = 0;
+        this.totalBlocked = 0;
+    }
+
+    /**
+     * Normalize URL path: strip query params, fragments, duplicate slashes
+     * @param {string} rawPath
+     * @returns {string}
+     */
+    normalizePath(rawPath) {
+        if (!rawPath || typeof rawPath !== 'string') return '/';
+        const cleaned = rawPath.split('?')[0].split('#')[0].trim();
+        // Replace multiple consecutive slashes with single slash, except root
+        return cleaned.replace(/\/+/g, '/') || '/';
+    }
+
+    /**
+     * Test if a path should be excluded from intelligence gathering
+     * @param {string} path
+     * @returns {boolean}
+     */
+    isExcluded(path) {
+        return this.excludePatterns.some(regex => regex.test(path));
+    }
+
+    /**
+     * Record a 404 occurrence
+     * @param {Object} event
+     * @param {string} event.path
+     * @param {string} [event.ip]
+     * @param {string} [event.userAgent]
+     * @returns {boolean} true if recorded, false if excluded
+     */
+    record404({ path, ip = '127.0.0.1', userAgent = '' }) {
+        const normalized = this.normalizePath(path);
+        if (this.isExcluded(normalized)) {
+            return false;
+        }
+
+        const now = new Date().toISOString();
+        this.totalHits++;
+
+        let entry = this.paths.get(normalized);
+        if (!entry) {
+            // Memory guard: prune least recently updated path if capacity reached
+            if (this.paths.size >= this.maxUniquePaths) {
+                const oldestPath = this.paths.keys().next().value;
+                if (oldestPath !== undefined) {
+                    this.paths.delete(oldestPath);
+                }
+            }
+
+            entry = {
+                count: 1,
+                firstSeen: now,
+                lastSeen: now,
+                uniqueIPs: new Set([ip]),
+                autoPromoted: false
+            };
+            this.paths.set(normalized, entry);
+        } else {
+            entry.count++;
+            entry.lastSeen = now;
+            // Cap Set to 500 IPs per path to avoid memory ballooning on massive botnet targets
+            if (entry.uniqueIPs.size < 500) {
+                entry.uniqueIPs.add(ip);
+            }
+        }
+
+        // Track scanner fingerprint if present
+        if (userAgent && typeof userAgent === 'string') {
+            const trimmedUA = userAgent.slice(0, 150).trim();
+            if (trimmedUA) {
+                const count = this.scannerFingerprints.get(trimmedUA) || 0;
+                if (this.scannerFingerprints.size < 1000 || this.scannerFingerprints.has(trimmedUA)) {
+                    this.scannerFingerprints.set(trimmedUA, count + 1);
+                }
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Increment total blocked threats counter
+     */
+    recordBlock() {
+        this.totalBlocked++;
+    }
+
+    /**
+     * Export serializable snapshot matching the approved 404_intelligence.json schema
+     * @param {Array<string>} [promotedPaths=[]] Current list of promoted honeypot paths
+     * @returns {Object}
+     */
+    getSnapshot(promotedPaths = []) {
+        const pathsObj = {};
+        const allUniqueIPs = new Set();
+
+        for (const [p, data] of this.paths.entries()) {
+            data.uniqueIPs.forEach(ip => allUniqueIPs.add(ip));
+            pathsObj[p] = {
+                count: data.count,
+                firstSeen: data.firstSeen,
+                lastSeen: data.lastSeen,
+                uniqueIPs: data.uniqueIPs.size,
+                autoPromoted: !!data.autoPromoted
+            };
+        }
+
+        const fingerprintsObj = {};
+        // Top 50 fingerprints sorted by count
+        const sortedFingerprints = Array.from(this.scannerFingerprints.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 50);
+        for (const [ua, count] of sortedFingerprints) {
+            fingerprintsObj[ua] = count;
+        }
+
+        return {
+            version: 1,
+            lastSaved: new Date().toISOString(),
+            startedAt: this.startedAt,
+            summary: {
+                totalHits: this.totalHits,
+                uniquePaths: this.paths.size,
+                uniqueIPs: allUniqueIPs.size,
+                totalBlocked: this.totalBlocked
+            },
+            paths: pathsObj,
+            promotedPaths: Array.from(new Set(promotedPaths)),
+            scannerFingerprints: fingerprintsObj
+        };
+    }
+
+    /**
+     * Restore state from a previously saved snapshot
+     * @param {Object} snapshot
+     * @returns {number} Number of restored paths
+     */
+    restoreFromSnapshot(snapshot) {
+        if (!snapshot || typeof snapshot !== 'object') return 0;
+
+        if (snapshot.startedAt) {
+            this.startedAt = snapshot.startedAt;
+        }
+        if (snapshot.summary) {
+            this.totalHits = Number(snapshot.summary.totalHits) || 0;
+            this.totalBlocked = Number(snapshot.summary.totalBlocked) || 0;
+        }
+
+        let restoredPaths = 0;
+        if (snapshot.paths && typeof snapshot.paths === 'object') {
+            for (const [p, data] of Object.entries(snapshot.paths)) {
+                if (restoredPaths >= this.maxUniquePaths) break;
+                this.paths.set(p, {
+                    count: Number(data.count) || 1,
+                    firstSeen: data.firstSeen || new Date().toISOString(),
+                    lastSeen: data.lastSeen || new Date().toISOString(),
+                    uniqueIPs: new Set(new Array(Math.min(50, Number(data.uniqueIPs) || 1)).fill('restored')),
+                    autoPromoted: !!data.autoPromoted
+                });
+                restoredPaths++;
+            }
+        }
+
+        if (snapshot.scannerFingerprints && typeof snapshot.scannerFingerprints === 'object') {
+            for (const [ua, count] of Object.entries(snapshot.scannerFingerprints)) {
+                this.scannerFingerprints.set(ua, Number(count) || 1);
+            }
+        }
+
+        return restoredPaths;
+    }
+
+    /**
+     * Diagnostic and administrative stats
+     */
+    getStats() {
+        return {
+            totalHits: this.totalHits,
+            uniquePaths: this.paths.size,
+            maxUniquePaths: this.maxUniquePaths,
+            totalBlocked: this.totalBlocked,
+            fingerprintCount: this.scannerFingerprints.size,
+            startedAt: this.startedAt
+        };
+    }
+}

package/src/intelligence/filePersistence.js

@@ -0,0 +1,226 @@
+/**
+ * Single-File Deduplicated Persistence for 404 Intelligence
+ * Atomically writes intelligence snapshots to data/shield/404_intelligence.json
+ * and restores learned patterns upon application startup.
+ *
+ * @author ChdLabs
+ * @license MIT
+ */
+import fs from 'fs';
+import path from 'path';
+
+export class FilePersistence {
+    /**
+     * @param {Object} options
+     * @param {boolean} [options.enabled=true]
+     * @param {string} [options.directory='./data/shield'] Relative to process.cwd() or absolute
+     * @param {string} [options.filename='404_intelligence.json']
+     * @param {number} [options.intervalMs=3600000] Save interval in ms (default: 1 hour)
+     * @param {boolean} [options.loadOnStartup=true] Read file on startup to restore state
+     * @param {Object} options.collector IntelligenceCollector instance
+     * @param {Object} options.honeypot Honeypot instance
+     * @param {Object} [options.autoPromoter] AutoPromoter instance
+     * @param {Object} [options.logger=console]
+     */
+    constructor({
+        enabled = true,
+        directory = './data/shield',
+        filename = '404_intelligence.json',
+        intervalMs = 60 * 60 * 1000,
+        loadOnStartup = true,
+        collector,
+        honeypot,
+        autoPromoter = null,
+        logger = console
+    } = {}) {
+        this.enabled = enabled;
+        this.intervalMs = Math.max(5000, intervalMs);
+        this.loadOnStartup = loadOnStartup;
+        this.collector = collector;
+        this.honeypot = honeypot;
+        this.autoPromoter = autoPromoter;
+        this.logger = logger;
+        this.timer = null;
+        this.lastSaved = null;
+        this.lastError = null;
+
+        // Secure path resolution & sandboxing against path traversal (CWE-22)
+        const baseDir = path.resolve(process.cwd(), directory);
+        const safeFilename = path.basename(filename || '404_intelligence.json');
+        const targetPath = path.resolve(baseDir, safeFilename);
+
+        // Verify directory boundary with trailing separator
+        const sandboxedBoundary = baseDir.endsWith(path.sep) ? baseDir : baseDir + path.sep;
+        if (!targetPath.startsWith(sandboxedBoundary) && targetPath !== baseDir) {
+            throw new Error(`[SHIELD SECURITY] Target path "${targetPath}" escapes permitted directory "${baseDir}"`);
+        }
+
+        this.baseDir = baseDir;
+        this.filePath = targetPath;
+        this.tempFilePath = `${targetPath}.tmp`;
+
+        // Ensure directory exists
+        if (this.enabled && !fs.existsSync(this.baseDir)) {
+            try {
+                fs.mkdirSync(this.baseDir, { recursive: true });
+            } catch (err) {
+                this.logger.warn?.(`[SHIELD] Warning: Could not create directory ${this.baseDir}: ${err.message}`);
+            }
+        }
+    }
+
+    /**
+     * Read and restore snapshot from disk on startup
+     * @returns {boolean} true if state was restored
+     */
+    load() {
+        if (!this.enabled || !this.loadOnStartup) {
+            return false;
+        }
+
+        try {
+            if (!fs.existsSync(this.filePath)) {
+                this.logger.info?.(`[SHIELD] No previous 404 intelligence file found at "${this.filePath}". Starting fresh.`);
+                return false;
+            }
+
+            const rawData = fs.readFileSync(this.filePath, 'utf8');
+            const parsed = JSON.parse(rawData);
+
+            // Restore collector paths & fingerprints
+            const restoredCount = this.collector.restoreFromSnapshot(parsed);
+
+            // Restore promoted paths into honeypot
+            let restoredTraps = 0;
+            if (parsed.promotedPaths && Array.isArray(parsed.promotedPaths)) {
+                for (const route of parsed.promotedPaths) {
+                    if (this.honeypot && typeof route === 'string') {
+                        this.honeypot.addRoute(route, 'restored_from_file');
+                        restoredTraps++;
+                    }
+                    if (this.autoPromoter) {
+                        this.autoPromoter.promotedHistory.add(route);
+                    }
+                }
+            }
+
+            this.logger.info?.(`[SHIELD] Restored ${restoredCount} intelligence paths and ${restoredTraps} promoted traps from "${this.filePath}"`);
+            return true;
+        } catch (err) {
+            this.lastError = err.message;
+            this.logger.warn?.(`[SHIELD] Warning: Failed to parse 404 intelligence file (${err.message}). Starting with clean state.`);
+            return false;
+        }
+    }
+
+    /**
+     * Atomically save intelligence snapshot to disk
+     * @returns {boolean} true if successfully written
+     */
+    save() {
+        if (!this.enabled || !this.collector) {
+            return false;
+        }
+
+        try {
+            // First run auto-promoter evaluation to capture newly qualified traps
+            if (this.autoPromoter) {
+                this.autoPromoter.evaluate();
+            }
+
+            const promotedPaths = this.honeypot ? this.honeypot.getPromotedRoutes() : [];
+            const snapshot = this.collector.getSnapshot(promotedPaths);
+            const serialized = JSON.stringify(snapshot, null, 2);
+
+            // Ensure destination directory exists
+            if (!fs.existsSync(this.baseDir)) {
+                fs.mkdirSync(this.baseDir, { recursive: true });
+            }
+
+            // Atomic write: write to temp file then rename
+            fs.writeFileSync(this.tempFilePath, serialized, 'utf8');
+            fs.renameSync(this.tempFilePath, this.filePath);
+
+            this.lastSaved = new Date().toISOString();
+            this.lastError = null;
+            return true;
+        } catch (err) {
+            this.lastError = err.message;
+            this.logger.error?.(`[SHIELD] Error saving 404 intelligence to "${this.filePath}": ${err.message}`);
+            // Cleanup temp file if stranded
+            try {
+                if (fs.existsSync(this.tempFilePath)) {
+                    fs.unlinkSync(this.tempFilePath);
+                }
+            } catch {
+                // Ignore cleanup errors
+            }
+            return false;
+        }
+    }
+
+    /**
+     * Start periodic auto-save timer
+     */
+    start() {
+        if (!this.enabled) return;
+
+        // Perform initial load if configured
+        if (this.loadOnStartup) {
+            this.load();
+        }
+
+        if (this.timer) {
+            clearInterval(this.timer);
+        }
+
+        this.timer = setInterval(() => {
+            this.save();
+        }, this.intervalMs);
+
+        // Do not hold Node process open if everything else shuts down
+        if (typeof this.timer.unref === 'function') {
+            this.timer.unref();
+        }
+    }
+
+    /**
+     * Stop periodic timer and perform final graceful flush
+     */
+    async stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+        if (this.enabled) {
+            this.save();
+        }
+    }
+
+    /**
+     * Get persistence health and diagnostics
+     */
+    getStatus() {
+        let fileExists = false;
+        let fileSize = 0;
+        try {
+            if (fs.existsSync(this.filePath)) {
+                fileExists = true;
+                const stat = fs.statSync(this.filePath);
+                fileSize = stat.size;
+            }
+        } catch {
+            // Ignore
+        }
+
+        return {
+            enabled: this.enabled,
+            filePath: this.filePath,
+            intervalMs: this.intervalMs,
+            lastSaved: this.lastSaved,
+            lastError: this.lastError,
+            fileExists,
+            fileSize
+        };
+    }
+}

package/src/middleware/costThrottle.js

@@ -0,0 +1,75 @@
+/**
+ * Denial-of-Wallet Cost Throttle Middleware Decorator
+ * Throttles expensive routes (e.g. AI generation, batch operations)
+ * by weighted cost units rather than raw request counts.
+ *
+ * @author ChdLabs
+ * @license MIT
+ */
+
+export class CostThrottle {
+    constructor() {
+        // ip -> { currentCost: number, windowStart: number }
+        this.clientCosts = new Map();
+
+        // Prune expired buckets every 2 minutes
+        this.cleanupTimer = setInterval(() => {
+            const now = Date.now();
+            for (const [ip, record] of this.clientCosts.entries()) {
+                if (now - record.windowStart > 120_000) {
+                    this.clientCosts.delete(ip);
+                }
+            }
+        }, 120_000);
+
+        if (typeof this.cleanupTimer.unref === 'function') {
+            this.cleanupTimer.unref();
+        }
+    }
+
+    /**
+     * Create a route-level cost throttle middleware
+     * @param {Object} options
+     * @param {number} [options.weight=1] Cost units consumed per request
+     * @param {number} [options.maxCost=100] Maximum units allowed per window
+     * @param {number} [options.windowMs=60000] Sliding window in ms (default: 1 min)
+     */
+    create({ weight = 1, maxCost = 100, windowMs = 60_000 } = {}) {
+        return (req, res, next) => {
+            const ip = (req.headers['x-forwarded-for']?.split(',')[0]?.trim()) ||
+                       req.socket?.remoteAddress ||
+                       req.ip ||
+                       '127.0.0.1';
+
+            const now = Date.now();
+            let record = this.clientCosts.get(ip);
+
+            if (!record || (now - record.windowStart > windowMs)) {
+                record = { currentCost: weight, windowStart: now };
+                this.clientCosts.set(ip, record);
+                return next();
+            }
+
+            if (record.currentCost + weight > maxCost) {
+                const retryAfterSec = Math.ceil((windowMs - (now - record.windowStart)) / 1000);
+                res.setHeader('Retry-After', retryAfterSec);
+                return res.status(429).json({
+                    error: 'Rate limit exceeded: Cost budget exhausted',
+                    code: 'COST_BUDGET_EXCEEDED',
+                    retryAfter: retryAfterSec
+                });
+            }
+
+            record.currentCost += weight;
+            next();
+        };
+    }
+
+    destroy() {
+        if (this.cleanupTimer) {
+            clearInterval(this.cleanupTimer);
+            this.cleanupTimer = null;
+        }
+        this.clientCosts.clear();
+    }
+}