imperium-crawl 2.3.1 → 2.5.1

package/dist/utils/fetcher.js

@@ -1,7 +1,8 @@
 import { smartFetch, StealthError } from "../stealth/index.js";
 import { isAllowed } from "./robots.js";
 import { getDomain } from "./url.js";
-import { DEFAULT_CONCURRENCY } from "../constants.js";
+import { DEFAULT_CONCURRENCY, DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS } from "../constants.js";
+import { getKnowledgeEngine } from "../knowledge/index.js";
 // ── Concurrency Limiter ──
 export class ConcurrencyLimiter {
     maxConcurrent;
@@ -30,22 +31,39 @@ export const defaultLimiter = new ConcurrencyLimiter();
 const CIRCUIT_FAILURE_THRESHOLD = 5;
 const CIRCUIT_OPEN_DURATION_MS = 60_000;
 const CIRCUIT_PROBE_SUCCESSES = 3;
+// Domain-level circuit: higher threshold — only opens when multiple endpoints fail
+const DOMAIN_CIRCUIT_FAILURE_THRESHOLD = 10;
 const circuits = new Map();
 // Periodic cleanup: remove closed circuits idle for >1 hour
 const CIRCUIT_STALE_MS = 3_600_000;
 setInterval(() => {
     const now = Date.now();
-    for (const [domain, circuit] of circuits) {
+    for (const [key, circuit] of circuits) {
         if (now - circuit.lastAccessed > CIRCUIT_STALE_MS) {
-            circuits.delete(domain);
+            circuits.delete(key);
         }
     }
 }, 300_000).unref();
-function getCircuit(domain) {
-    let circuit = circuits.get(domain);
+/**
+ * Get circuit breaker key for a URL.
+ * Uses domain + first 2 path segments for endpoint-level granularity.
+ */
+function getCircuitKey(url) {
+    try {
+        const parsed = new URL(url);
+        const pathParts = parsed.pathname.split("/").filter(Boolean);
+        const pathPrefix = pathParts.slice(0, 2).join("/");
+        return pathPrefix ? `${parsed.hostname}/${pathPrefix}` : parsed.hostname;
+    }
+    catch {
+        return getDomain(url);
+    }
+}
+function getCircuit(key) {
+    let circuit = circuits.get(key);
     if (!circuit) {
         circuit = { state: "closed", failures: 0, openedAt: 0, probeSuccesses: 0, lastAccessed: Date.now() };
-        circuits.set(domain, circuit);
+        circuits.set(key, circuit);
     }
     circuit.lastAccessed = Date.now();
     // Check if open circuit should transition to half-open
@@ -55,8 +73,8 @@ function getCircuit(domain) {
     }
     return circuit;
 }
-function recordSuccess(domain) {
-    const circuit = getCircuit(domain);
+function recordSuccess(key) {
+    const circuit = getCircuit(key);
     if (circuit.state === "half-open") {
         circuit.probeSuccesses++;
         if (circuit.probeSuccesses >= CIRCUIT_PROBE_SUCCESSES) {
@@ -69,8 +87,8 @@ function recordSuccess(domain) {
         circuit.failures = 0;
     }
 }
-function recordFailure(domain) {
-    const circuit = getCircuit(domain);
+function recordFailure(key) {
+    const circuit = getCircuit(key);
     // Half-open probe failed → immediately reopen circuit
     if (circuit.state === "half-open") {
         circuit.state = "open";
@@ -84,14 +102,83 @@ function recordFailure(domain) {
         circuit.openedAt = Date.now();
     }
 }
+/**
+ * Check domain-level circuit: opens when 3+ endpoint circuits are open for this domain.
+ */
+function isDomainCircuitOpen(domain) {
+    let openEndpoints = 0;
+    for (const [key, circuit] of circuits) {
+        if (key.startsWith(domain) && circuit.state === "open") {
+            openEndpoints++;
+        }
+    }
+    return openEndpoints >= 3;
+}
 // Exported for testing
-export { circuits, getCircuit, recordSuccess, recordFailure, CIRCUIT_FAILURE_THRESHOLD, CIRCUIT_OPEN_DURATION_MS, CIRCUIT_PROBE_SUCCESSES, CIRCUIT_STALE_MS };
+export { circuits, getCircuit, getCircuitKey, recordSuccess, recordFailure, isDomainCircuitOpen, CIRCUIT_FAILURE_THRESHOLD, CIRCUIT_OPEN_DURATION_MS, CIRCUIT_PROBE_SUCCESSES, CIRCUIT_STALE_MS };
 // ── Exponential Backoff with Full Jitter (AWS pattern) ──
 const BACKOFF_BASE_MS = 1000;
 const BACKOFF_CAP_MS = 30_000;
-function fullJitterBackoff(attempt) {
+const RATE_LIMIT_EXTRA_JITTER_MS = 10_000; // Extra jitter for 429 responses
+function fullJitterBackoff(attempt, is429 = false) {
     const expDelay = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * Math.pow(2, attempt));
-    return Math.random() * expDelay;
+    const baseJitter = Math.random() * expDelay;
+    // On 429, add extra random jitter to avoid thundering herd
+    if (is429) {
+        return baseJitter + 5000 + Math.random() * RATE_LIMIT_EXTRA_JITTER_MS;
+    }
+    return baseJitter;
+}
+// ── Per-Domain Rate Limiter ──
+const DEFAULT_DOMAIN_RATE_MS = parseInt(process.env.DOMAIN_RATE_LIMIT_MS || "500", 10);
+class DomainThrottle {
+    lastRequest = new Map();
+    defaultDelay;
+    constructor(defaultDelayMs = DEFAULT_DOMAIN_RATE_MS) {
+        this.defaultDelay = defaultDelayMs;
+    }
+    /**
+     * Wait until enough time has passed since the last request to this domain.
+     * Uses knowledge engine's safe_rate_limit if available, else default delay.
+     */
+    async throttle(domain, knowledgeDelayMs) {
+        const delay = knowledgeDelayMs ?? this.defaultDelay;
+        if (delay <= 0)
+            return;
+        const now = Date.now();
+        const last = this.lastRequest.get(domain) ?? 0;
+        const elapsed = now - last;
+        if (elapsed < delay) {
+            await new Promise((r) => setTimeout(r, delay - elapsed));
+        }
+        this.lastRequest.set(domain, Date.now());
+    }
+}
+const domainThrottle = new DomainThrottle();
+/**
+ * Compute adaptive timeout based on knowledge engine data.
+ * Uses avg_response_time * 3 with a floor of DEFAULT_TIMEOUT_MS and ceiling of MAX_TIMEOUT_MS.
+ */
+function computeAdaptiveTimeout(avgResponseTimeMs) {
+    if (!avgResponseTimeMs || avgResponseTimeMs <= 0)
+        return DEFAULT_TIMEOUT_MS;
+    return Math.min(MAX_TIMEOUT_MS, Math.max(DEFAULT_TIMEOUT_MS, avgResponseTimeMs * 3));
+}
+/**
+ * Determine the escalated stealth level for a retry attempt.
+ * attempt 0: user's level, attempt 1: level+1, attempt 2: L3
+ */
+function getEscalatedLevel(baseLevel, attempt, lastError) {
+    // If last failure was a StealthError with detected anti-bot, jump to L3
+    if (lastError instanceof StealthError && lastError.antiBotSystem) {
+        return 3;
+    }
+    const base = baseLevel || 1;
+    if (attempt === 0)
+        return base;
+    if (attempt === 1)
+        return Math.min(base + 1, 3);
+    return 3; // attempt >= 2 → always L3
 }
 export async function fetchPage(url, options) {
     const respectRobots = options?.respectRobots ?? (process.env.RESPECT_ROBOTS !== "false");
@@ -101,34 +188,77 @@ export async function fetchPage(url, options) {
             throw new Error(`URL blocked by robots.txt: ${url}`);
         }
     }
-    // Circuit breaker check
+    // Per-endpoint circuit breaker check
     const domain = getDomain(url);
-    const circuit = getCircuit(domain);
+    const circuitKey = getCircuitKey(url);
+    const circuit = getCircuit(circuitKey);
     if (circuit.state === "open") {
-        throw new Error(`Circuit breaker open for ${domain} — too many consecutive failures. Retry after cooldown.`);
+        throw new Error(`Circuit breaker open for endpoint ${circuitKey} — too many consecutive failures. Retry after cooldown.`);
+    }
+    // Domain-level circuit check (opens when 3+ endpoints are broken)
+    if (isDomainCircuitOpen(domain)) {
+        throw new Error(`Circuit breaker open for domain ${domain} — multiple endpoints failing. Retry after cooldown.`);
     }
+    // Per-domain rate limiting — use knowledge engine's safe_rate_limit if available
+    const engine = getKnowledgeEngine();
+    const knowledge = await engine.get(domain);
+    const knowledgeDelayMs = knowledge?.safe_rate_limit
+        ? Math.round(60_000 / knowledge.safe_rate_limit)
+        : undefined;
+    await domainThrottle.throttle(domain, knowledgeDelayMs);
+    // ── Adaptive timeout from knowledge engine ──
+    const adaptiveTimeout = computeAdaptiveTimeout(knowledge?.avg_response_time_ms);
+    const timeout = options?.timeout || adaptiveTimeout;
     const retries = options?.retries ?? 2;
     let lastError;
+    let lastHttpStatus = 0;
     for (let attempt = 0; attempt <= retries; attempt++) {
         try {
-            const result = await smartFetch(url, options);
-            recordSuccess(domain);
+            // ── Smart retry: escalate stealth level on each attempt ──
+            const escalatedLevel = getEscalatedLevel(options?.forceLevel, attempt, lastError);
+            const attemptOptions = {
+                ...options,
+                timeout,
+                // On retry, escalate stealth level (unless user forced a specific level)
+                forceLevel: attempt > 0 && !options?.forceLevel ? escalatedLevel : options?.forceLevel,
+            };
+            const result = await smartFetch(url, attemptOptions);
+            recordSuccess(circuitKey);
+            // Feed successful strategy back to knowledge engine
+            if (attempt > 0 && result.level > 1) {
+                engine.record({
+                    url, domain,
+                    levelUsed: result.level,
+                    success: true,
+                    responseTimeMs: 0, // Already recorded by smartFetch
+                    antiBotSystem: result.antiBotSystem || null,
+                    captchaType: result.captchaSolved ? "detected" : null,
+                    proxyUsed: !!result.proxyUsed,
+                    blocked: false,
+                    httpStatus: result.status,
+                });
+            }
             return result;
         }
         catch (err) {
             lastError = err instanceof Error ? err : new Error(String(err));
-            recordFailure(domain);
-            // Check if circuit just opened
-            const updatedCircuit = getCircuit(domain);
+            recordFailure(circuitKey);
+            // Track HTTP status for backoff decisions
+            if (err instanceof StealthError) {
+                lastHttpStatus = err.httpStatus;
+            }
+            // Check if endpoint circuit just opened
+            const updatedCircuit = getCircuit(circuitKey);
             if (updatedCircuit.state === "open") {
                 // Enrich error message with StealthError info if available
                 const detail = err instanceof StealthError
                     ? `L${err.lastLevel} HTTP ${err.httpStatus}${err.antiBotSystem ? ` [${err.antiBotSystem}]` : ""}`
                     : "";
-                throw new Error(`Circuit breaker opened for ${domain}${detail ? ` (${detail})` : ""}: ${lastError.message}`);
+                throw new Error(`Circuit breaker opened for ${circuitKey}${detail ? ` (${detail})` : ""}: ${lastError.message}`);
             }
             if (attempt < retries) {
-                const delay = fullJitterBackoff(attempt);
+                const is429 = lastHttpStatus === 429;
+                const delay = fullJitterBackoff(attempt, is429);
                 await new Promise((r) => setTimeout(r, delay));
             }
         }

package/dist/utils/fetcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAyC,MAAM,qBAAqB,CAAC;AACtG,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEtD,4BAA4B;AAE5B,MAAM,OAAO,kBAAkB;IAIT;IAHZ,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAsB,EAAE,CAAC;IAEtC,YAAoB,gBAAwB,mBAAmB;QAA3C,kBAAa,GAAb,aAAa,CAA8B;IAAG,CAAC;IAEnE,KAAK,CAAC,GAAG,CAAI,EAAoB;QAC/B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,kBAAkB,EAAE,CAAC;AAcvD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;AAEnD,4DAA4D;AAC5D,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,WAAW,CAAC,GAAG,EAAE;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzC,IAAI,GAAG,GAAG,OAAO,CAAC,YAAY,GAAG,gBAAgB,EAAE,CAAC;YAClD,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AAEpB,SAAS,UAAU,CAAC,MAAc;IAChC,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACrG,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,uDAAuD;IACvD,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,wBAAwB,EAAE,CAAC;QAC1F,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC;QAC5B,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,cAAc,IAAI,uBAAuB,EAAE,CAAC;YACtD,kBAAkB;YAClB,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,sDAAsD;IACtD,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,GAAG,yBAAyB,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,QAAQ,EAAE,CAAC;IACnB,IAAI,OAAO,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,uBAAuB;AACvB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAE9J,2DAA2D;AAE3D,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,SAAS,iBAAiB,CAAC,OAAe;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC;AAClC,CAAC;AASD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAA2B;IACtE,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC;IAEzF,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,yDAAyD,CAAC,CAAC;IAC/G,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IACtC,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9C,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,aAAa,CAAC,MAAM,CAAC,CAAC;YAEtB,+BAA+B;YAC/B,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,cAAc,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACpC,2DAA2D;gBAC3D,MAAM,MAAM,GAAG,GAAG,YAAY,YAAY;oBACxC,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,SAAS,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACjG,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/G,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAU,CAAC;AACnB,CAAC"}
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAA4D,MAAM,qBAAqB,CAAC;AACzH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,4BAA4B;AAE5B,MAAM,OAAO,kBAAkB;IAIT;IAHZ,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAsB,EAAE,CAAC;IAEtC,YAAoB,gBAAwB,mBAAmB;QAA3C,kBAAa,GAAb,aAAa,CAA8B;IAAG,CAAC;IAEnE,KAAK,CAAC,GAAG,CAAI,EAAoB;QAC/B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,kBAAkB,EAAE,CAAC;AAcvD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAClC,mFAAmF;AACnF,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAE5C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;AAEnD,4DAA4D;AAC5D,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,WAAW,CAAC,GAAG,EAAE;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,GAAG,GAAG,OAAO,CAAC,YAAY,GAAG,gBAAgB,EAAE,CAAC;YAClD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AAEpB;;;GAGG;AACH,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,UAAU,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACrG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,uDAAuD;IACvD,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,wBAAwB,EAAE,CAAC;QAC1F,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC;QAC5B,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,cAAc,IAAI,uBAAuB,EAAE,CAAC;YACtD,kBAAkB;YAClB,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAChC,sDAAsD;IACtD,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,GAAG,yBAAyB,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,QAAQ,EAAE,CAAC;IACnB,IAAI,OAAO,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAc;IACzC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YACvD,aAAa,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,aAAa,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED,uBAAuB;AACvB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAElM,2DAA2D;AAE3D,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,0BAA0B,GAAG,MAAM,CAAC,CAAC,iCAAiC;AAE5E,SAAS,iBAAiB,CAAC,OAAe,EAAE,KAAK,GAAG,KAAK;IACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC;IAC5C,2DAA2D;IAC3D,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,UAAU,GAAG,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,0BAA0B,CAAC;IACxE,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,gCAAgC;AAEhC,MAAM,sBAAsB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;AAEvF,MAAM,cAAc;IACV,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,YAAY,CAAS;IAE7B,YAAY,iBAAyB,sBAAsB;QACzD,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;IACrC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAc,EAAE,gBAAyB;QACtD,MAAM,KAAK,GAAG,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC;QACpD,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO;QAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC;QAE3B,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;AAS5C;;;GAGG;AACH,SAAS,sBAAsB,CAAC,iBAAqC;IACnE,IAAI,CAAC,iBAAiB,IAAI,iBAAiB,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC5E,OAAO,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC;AACvF,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CACxB,SAAmC,EACnC,OAAe,EACf,SAA4B;IAE5B,wEAAwE;IACxE,IAAI,SAAS,YAAY,YAAY,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;QACjE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,IAAI,GAAG,SAAS,IAAI,CAAC,CAAC;IAC5B,IAAI,OAAO,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/B,IAAI,OAAO,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAiB,CAAC;IAChE,OAAO,CAAC,CAAC,CAAC,2BAA2B;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAA2B;IACtE,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC;IAEzF,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACvC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,qCAAqC,UAAU,yDAAyD,CAAC,CAAC;IAC5H,CAAC;IAED,kEAAkE;IAClE,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,mCAAmC,MAAM,sDAAsD,CAAC,CAAC;IACnH,CAAC;IAED,iFAAiF;IACjF,MAAM,MAAM,GAAG,kBAAkB,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,gBAAgB,GAAG,SAAS,EAAE,eAAe;QACjD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,eAAe,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAExD,+CAA+C;IAC/C,MAAM,eAAe,GAAG,sBAAsB,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;IAChF,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,eAAe,CAAC;IAEpD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IACtC,IAAI,SAA4B,CAAC;IACjC,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,4DAA4D;YAC5D,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAClF,MAAM,cAAc,GAAmB;gBACrC,GAAG,OAAO;gBACV,OAAO;gBACP,yEAAyE;gBACzE,UAAU,EAAE,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,EAAE,UAAU;aACvF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;YACrD,aAAa,CAAC,UAAU,CAAC,CAAC;YAE1B,oDAAoD;YACpD,IAAI,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,CAAC,MAAM,CAAC;oBACZ,GAAG,EAAE,MAAM;oBACX,SAAS,EAAE,MAAM,CAAC,KAAK;oBACvB,OAAO,EAAE,IAAI;oBACb,cAAc,EAAE,CAAC,EAAE,iCAAiC;oBACpD,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,IAAI;oBAC3C,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;oBACrD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS;oBAC7B,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,MAAM,CAAC,MAAM;iBAC1B,CAAC,CAAC;YACL,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,aAAa,CAAC,UAAU,CAAC,CAAC;YAE1B,0CAA0C;YAC1C,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;gBAChC,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC;YAClC,CAAC;YAED,wCAAwC;YACxC,MAAM,cAAc,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YAC9C,IAAI,cAAc,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACpC,2DAA2D;gBAC3D,MAAM,MAAM,GAAG,GAAG,YAAY,YAAY;oBACxC,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,SAAS,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACjG,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YACnH,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,cAAc,KAAK,GAAG,CAAC;gBACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAChD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAU,CAAC;AACnB,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "imperium-crawl",
-  "version": "2.3.1",
-  "description": "Open-source CLI tool for web scraping, crawling, search, and custom skills",
+  "version": "2.5.1",
+  "description": "39-tool open-source CLI for web scraping, PDF extraction, content monitoring, reusable browser flows, RSS aggregation, and custom skills. Zero API keys for core tools.",
   "type": "module",
   "bin": {
     "imperium-crawl": "dist/index.js",
@@ -11,7 +11,10 @@
   "exports": {
     ".": "./dist/index.js",
     "./tools/*": "./dist/tools/*.js",
-    "./stealth/*": "./dist/stealth/*.js"
+    "./stealth/*": "./dist/stealth/*.js",
+    "./sessions": "./dist/sessions/index.js",
+    "./sessions/*": "./dist/sessions/*.js",
+    "./utils/*": "./dist/utils/*.js"
   },
   "files": [
     "dist",
@@ -23,7 +26,9 @@
     "start": "node dist/index.js",
     "test": "vitest run",
     "test:watch": "vitest",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "npm run build",
+    "autoresearch": "tsx autoresearch/eval.ts",
+    "autoresearch:baseline": "tsx autoresearch/eval.ts --baseline --verbose"
   },
   "keywords": [
     "scraping",
@@ -31,7 +36,15 @@
     "web-search",
     "brave-search",
     "firecrawl",
-    "cli"
+    "cli",
+    "pdf-extract",
+    "web-monitoring",
+    "url-watch",
+    "content-diff",
+    "intelligence-digest",
+    "browser-workflows",
+    "workflow-recorder",
+    "flow-api"
   ],
   "author": "ImperiumTech",
   "license": "MIT",
@@ -52,6 +65,7 @@
     "normalize-url": "^8.1.1",
     "ora": "^8.2.0",
     "p-queue": "^8.1.1",
+    "pdfjs-dist": "^4.0.379",
     "playwright": "1.52",
     "robots-parser": "^3.0.1",
     "rss-parser": "^3.13.0",