@detiq/crawler 1.1.9 → 1.1.10

package/dist/proxy.js

@@ -1,191 +1,165 @@
-/**
- * Free proxy auto-selection for bot-protection bypass.
- *
- * Source: proxifly/free-proxy-list (MIT) via jsDelivr CDN.
- * These are datacenter proxies — they bypass simple IP blocks but NOT
- * full Cloudflare Bot Manager / Akamai. For Cloudflare-protected sites,
- * users need a residential proxy (Bright Data, Oxylabs, etc.).
- */
-import { createConnection } from 'net';
-const PROXY_LIST_URL = 'https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json';
-/** Fetch and filter proxy list. Returns http proxies sorted by speed (fastest first). */
+import { createConnection } from "net";
+const PROXY_LIST_URL = "https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json";
 async function fetchCandidates() {
-    const res = await fetch(PROXY_LIST_URL, {
-        signal: AbortSignal.timeout(10_000),
-        headers: { 'Accept': 'application/json' },
-    });
-    if (!res.ok)
-        throw new Error(`proxy list fetch failed: ${res.status}`);
-    const raw = await res.json();
-    return raw
-        .filter((p) => p.protocols?.includes('http') && p.port > 0 && (p.upTime ?? 0) >= 30)
-        .sort((a, b) => (a.speed ?? 99999) - (b.speed ?? 99999))
-        .slice(0, 50); // test at most 50 candidates
+  const res = await fetch(PROXY_LIST_URL, {
+    signal: AbortSignal.timeout(1e4),
+    headers: { "Accept": "application/json" }
+  });
+  if (!res.ok) throw new Error(`proxy list fetch failed: ${res.status}`);
+  const raw = await res.json();
+  return raw.filter((p) => p.protocols?.includes("http") && p.port > 0 && (p.upTime ?? 0) >= 30).sort((a, b) => (a.speed ?? 99999) - (b.speed ?? 99999)).slice(0, 50);
 }
-/** TCP-level proxy liveness check via HTTP CONNECT tunnel. No extra dependencies. */
 function testProxy(ip, port, timeoutMs) {
-    return new Promise((resolve) => {
-        let settled = false;
-        const settle = (ok) => {
-            if (settled)
-                return;
-            settled = true;
-            resolve(ok);
-            try {
-                sock.destroy();
-            }
-            catch { /* ignore */ }
-        };
-        const timer = setTimeout(() => settle(false), timeoutMs);
-        const sock = createConnection({ host: ip, port, timeout: timeoutMs });
-        sock.on('connect', () => {
-            sock.write('CONNECT example.com:80 HTTP/1.1\r\nHost: example.com:80\r\n\r\n');
-            sock.once('data', (data) => {
-                clearTimeout(timer);
-                const ok = /^HTTP\/1\.[01] 200/.test(data.toString());
-                settle(ok);
-            });
-        });
-        sock.on('error', () => { clearTimeout(timer); settle(false); });
-        sock.on('timeout', () => { clearTimeout(timer); settle(false); });
+  return new Promise((resolve) => {
+    let settled = false;
+    const settle = (ok) => {
+      if (settled) return;
+      settled = true;
+      resolve(ok);
+      try {
+        sock.destroy();
+      } catch {
+      }
+    };
+    const timer = setTimeout(() => settle(false), timeoutMs);
+    const sock = createConnection({ host: ip, port, timeout: timeoutMs });
+    sock.on("connect", () => {
+      sock.write("CONNECT example.com:80 HTTP/1.1\r\nHost: example.com:80\r\n\r\n");
+      sock.once("data", (data) => {
+        clearTimeout(timer);
+        const ok = /^HTTP\/1\.[01] 200/.test(data.toString());
+        settle(ok);
+      });
+    });
+    sock.on("error", () => {
+      clearTimeout(timer);
+      settle(false);
     });
+    sock.on("timeout", () => {
+      clearTimeout(timer);
+      settle(false);
+    });
+  });
 }
-/** Test `batchSize` proxies concurrently and return first working one. */
 async function testBatch(candidates, start, batchSize, timeoutMs) {
-    const batch = candidates.slice(start, start + batchSize);
-    if (batch.length === 0)
-        return null;
-    const results = await Promise.all(batch.map(async (p) => {
-        const ok = await testProxy(p.ip, p.port, timeoutMs);
-        return ok ? `http://${p.ip}:${p.port}` : null;
-    }));
-    return results.find((r) => r !== null) ?? null;
+  const batch = candidates.slice(start, start + batchSize);
+  if (batch.length === 0) return null;
+  const results = await Promise.all(
+    batch.map(async (p) => {
+      const ok = await testProxy(p.ip, p.port, timeoutMs);
+      return ok ? `http://${p.ip}:${p.port}` : null;
+    })
+  );
+  return results.find((r) => r !== null) ?? null;
 }
-/**
- * Pick a working free proxy. Tests in parallel batches of 10 with a 4s timeout.
- * Returns null if no working proxy found (not an error — crawl can proceed without proxy).
- */
-export async function pickFreeProxy() {
-    let candidates;
-    try {
-        candidates = await fetchCandidates();
-    }
-    catch (err) {
-        console.warn('[proxy] failed to fetch proxy list:', err?.message ?? err);
-        return null;
-    }
-    const BATCH = 10;
-    const TIMEOUT = 4_000;
-    for (let i = 0; i < candidates.length; i += BATCH) {
-        const found = await testBatch(candidates, i, BATCH, TIMEOUT);
-        if (found) {
-            console.info(`[proxy] selected free proxy: ${found} (tested ${i + BATCH} candidates)`);
-            return found;
-        }
-    }
-    console.warn('[proxy] no working free proxy found in list');
+async function pickFreeProxy() {
+  let candidates;
+  try {
+    candidates = await fetchCandidates();
+  } catch (err) {
+    console.warn("[proxy] failed to fetch proxy list:", err?.message ?? err);
     return null;
+  }
+  const BATCH = 10;
+  const TIMEOUT = 4e3;
+  for (let i = 0; i < candidates.length; i += BATCH) {
+    const found = await testBatch(candidates, i, BATCH, TIMEOUT);
+    if (found) {
+      console.info(`[proxy] selected free proxy: ${found} (tested ${i + BATCH} candidates)`);
+      return found;
+    }
+  }
+  console.warn("[proxy] no working free proxy found in list");
+  return null;
 }
-const TIER_ORDER = ['free', 'datacenter', 'residential', 'premium'];
+const TIER_ORDER = ["free", "datacenter", "residential", "premium"];
 function nextTier(current) {
-    const idx = TIER_ORDER.indexOf(current);
-    return TIER_ORDER[Math.min(idx + 1, TIER_ORDER.length - 1)];
+  const idx = TIER_ORDER.indexOf(current);
+  return TIER_ORDER[Math.min(idx + 1, TIER_ORDER.length - 1)];
 }
-/**
- * Tiered proxy manager with per-domain error histograms.
- *
- * Tracks success/failure per domain and automatically escalates to higher proxy tiers
- * on consecutive failures. Applies time-decay to probe returning to lower tiers.
- *
- * Usage:
- *   const mgr = new TieredProxyManager([{ tier: 'datacenter', urls: [...] }]);
- *   const proxy = await mgr.selectProxy('api.salesforce.com');
- *   // ... make request ...
- *   mgr.recordResult('api.salesforce.com', success);
- */
-export class TieredProxyManager {
-    tiers = new Map();
-    domainHistory = new Map();
-    ESCALATE_AFTER = 3; // consecutive failures before escalation
-    DECAY_AFTER_MS = 30 * 60 * 1000; // 30 min: probe tier downgrade
-    constructor(configs) {
-        for (const cfg of configs) {
-            if (cfg.urls.length > 0)
-                this.tiers.set(cfg.tier, cfg.urls);
-        }
+class TieredProxyManager {
+  tiers = /* @__PURE__ */ new Map();
+  domainHistory = /* @__PURE__ */ new Map();
+  ESCALATE_AFTER = 3;
+  // consecutive failures before escalation
+  DECAY_AFTER_MS = 30 * 60 * 1e3;
+  // 30 min: probe tier downgrade
+  constructor(configs) {
+    for (const cfg of configs) {
+      if (cfg.urls.length > 0) this.tiers.set(cfg.tier, cfg.urls);
     }
-    getHistory(domain) {
-        if (!this.domainHistory.has(domain)) {
-            this.domainHistory.set(domain, {
-                tier: this.lowestAvailableTier(),
-                consecutiveFailures: 0,
-                lastEscalatedAt: 0,
-                totalRequests: 0,
-                successRate: 1,
-            });
-        }
-        return this.domainHistory.get(domain);
+  }
+  getHistory(domain) {
+    if (!this.domainHistory.has(domain)) {
+      this.domainHistory.set(domain, {
+        tier: this.lowestAvailableTier(),
+        consecutiveFailures: 0,
+        lastEscalatedAt: 0,
+        totalRequests: 0,
+        successRate: 1
+      });
     }
-    lowestAvailableTier() {
-        for (const t of TIER_ORDER) {
-            if (this.tiers.has(t))
-                return t;
-        }
-        return 'free';
+    return this.domainHistory.get(domain);
+  }
+  lowestAvailableTier() {
+    for (const t of TIER_ORDER) {
+      if (this.tiers.has(t)) return t;
     }
-    /** Pick a proxy URL for this domain, based on its tier history. */
-    selectProxy(domain) {
-        const history = this.getHistory(domain);
-        // Time-decay: if we've been on elevated tier for DECAY_AFTER_MS, try stepping down
-        if (history.consecutiveFailures === 0 && history.lastEscalatedAt > 0) {
-            const elapsed = Date.now() - history.lastEscalatedAt;
-            if (elapsed > this.DECAY_AFTER_MS) {
-                const currentIdx = TIER_ORDER.indexOf(history.tier);
-                if (currentIdx > 0) {
-                    const lowerTier = TIER_ORDER[currentIdx - 1];
-                    if (this.tiers.has(lowerTier)) {
-                        history.tier = lowerTier;
-                        history.lastEscalatedAt = 0;
-                    }
-                }
-            }
+    return "free";
+  }
+  /** Pick a proxy URL for this domain, based on its tier history. */
+  selectProxy(domain) {
+    const history = this.getHistory(domain);
+    if (history.consecutiveFailures === 0 && history.lastEscalatedAt > 0) {
+      const elapsed = Date.now() - history.lastEscalatedAt;
+      if (elapsed > this.DECAY_AFTER_MS) {
+        const currentIdx = TIER_ORDER.indexOf(history.tier);
+        if (currentIdx > 0) {
+          const lowerTier = TIER_ORDER[currentIdx - 1];
+          if (this.tiers.has(lowerTier)) {
+            history.tier = lowerTier;
+            history.lastEscalatedAt = 0;
+          }
         }
-        const urls = this.tiers.get(history.tier);
-        if (!urls || urls.length === 0)
-            return null;
-        // Round-robin within tier
-        const idx = history.totalRequests % urls.length;
-        return urls[idx];
+      }
     }
-    /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
-    recordResult(domain, success) {
-        const history = this.getHistory(domain);
-        history.totalRequests++;
-        if (success) {
-            history.consecutiveFailures = 0;
-            // Update EMA success rate
-            history.successRate = history.successRate * 0.9 + 0.1;
+    const urls = this.tiers.get(history.tier);
+    if (!urls || urls.length === 0) return null;
+    const idx = history.totalRequests % urls.length;
+    return urls[idx];
+  }
+  /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
+  recordResult(domain, success) {
+    const history = this.getHistory(domain);
+    history.totalRequests++;
+    if (success) {
+      history.consecutiveFailures = 0;
+      history.successRate = history.successRate * 0.9 + 0.1;
+    } else {
+      history.consecutiveFailures++;
+      history.successRate = history.successRate * 0.9;
+      if (history.consecutiveFailures >= this.ESCALATE_AFTER) {
+        const elevated = nextTier(history.tier);
+        if (elevated !== history.tier && this.tiers.has(elevated)) {
+          console.warn(`[proxy-tier] domain=${domain} escalating ${history.tier}\u2192${elevated} after ${history.consecutiveFailures} failures`);
+          history.tier = elevated;
+          history.lastEscalatedAt = Date.now();
+          history.consecutiveFailures = 0;
         }
-        else {
-            history.consecutiveFailures++;
-            history.successRate = history.successRate * 0.9;
-            if (history.consecutiveFailures >= this.ESCALATE_AFTER) {
-                const elevated = nextTier(history.tier);
-                if (elevated !== history.tier && this.tiers.has(elevated)) {
-                    console.warn(`[proxy-tier] domain=${domain} escalating ${history.tier}→${elevated} after ${history.consecutiveFailures} failures`);
-                    history.tier = elevated;
-                    history.lastEscalatedAt = Date.now();
-                    history.consecutiveFailures = 0;
-                }
-            }
-        }
-    }
-    stats() {
-        return Array.from(this.domainHistory.entries()).map(([domain, h]) => ({
-            domain,
-            tier: h.tier,
-            successRate: Math.round(h.successRate * 100) / 100,
-            totalRequests: h.totalRequests,
-        }));
+      }
     }
+  }
+  stats() {
+    return Array.from(this.domainHistory.entries()).map(([domain, h]) => ({
+      domain,
+      tier: h.tier,
+      successRate: Math.round(h.successRate * 100) / 100,
+      totalRequests: h.totalRequests
+    }));
+  }
 }
+export {
+  TieredProxyManager,
+  pickFreeProxy
+};
+//# sourceMappingURL=proxy.js.map

package/dist/rate-limiter.js

@@ -1,71 +1,64 @@
-/**
- * Domain-aware rate limiter — ensures the crawler respects per-domain rate limits.
- *
- * Uses a token bucket algorithm per domain. When the bucket is empty, requests
- * wait until a token is available. Tokens refill at a configurable rate.
- *
- * Default: 2 requests/second per domain, burst of 5.
- * Can be configured per-domain via the rateMap (domain → requests/second).
- */
 function extractDomain(url) {
-    try {
-        return new URL(url).hostname;
-    }
-    catch {
-        return url;
-    }
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return url;
+  }
 }
-export class DomainRateLimiter {
-    buckets = new Map();
-    defaultRps;
-    burstSize;
-    rateMap;
-    constructor(opts = {}) {
-        this.defaultRps = opts.defaultRps ?? 2;
-        this.burstSize = opts.burstSize ?? 5;
-        this.rateMap = opts.rateMap ?? {};
-    }
-    getBucket(domain) {
-        if (!this.buckets.has(domain)) {
-            this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });
-        }
-        return this.buckets.get(domain);
-    }
-    refill(domain, bucket) {
-        const rps = this.rateMap[domain] ?? this.defaultRps;
-        const now = Date.now();
-        const elapsed = (now - bucket.lastRefill) / 1000;
-        bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);
-        bucket.lastRefill = now;
-    }
-    /** Wait until a token is available for the given URL's domain. */
-    async throttle(url) {
-        const domain = extractDomain(url);
-        const bucket = this.getBucket(domain);
-        this.refill(domain, bucket);
-        if (bucket.tokens >= 1) {
-            bucket.tokens -= 1;
-            return;
-        }
-        // Calculate wait time until next token
-        const rps = this.rateMap[domain] ?? this.defaultRps;
-        const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1000);
-        await new Promise(r => setTimeout(r, waitMs));
-        bucket.tokens = 0;
-        bucket.lastRefill = Date.now();
+class DomainRateLimiter {
+  buckets = /* @__PURE__ */ new Map();
+  defaultRps;
+  burstSize;
+  rateMap;
+  constructor(opts = {}) {
+    this.defaultRps = opts.defaultRps ?? 2;
+    this.burstSize = opts.burstSize ?? 5;
+    this.rateMap = opts.rateMap ?? {};
+  }
+  getBucket(domain) {
+    if (!this.buckets.has(domain)) {
+      this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });
     }
-    /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
-    setRate(domain, rps) {
-        this.rateMap[domain] = rps;
-        const bucket = this.getBucket(domain);
-        bucket.tokens = 0; // drain immediately on rate change
+    return this.buckets.get(domain);
+  }
+  refill(domain, bucket) {
+    const rps = this.rateMap[domain] ?? this.defaultRps;
+    const now = Date.now();
+    const elapsed = (now - bucket.lastRefill) / 1e3;
+    bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);
+    bucket.lastRefill = now;
+  }
+  /** Wait until a token is available for the given URL's domain. */
+  async throttle(url) {
+    const domain = extractDomain(url);
+    const bucket = this.getBucket(domain);
+    this.refill(domain, bucket);
+    if (bucket.tokens >= 1) {
+      bucket.tokens -= 1;
+      return;
     }
-    stats() {
-        const result = {};
-        for (const [domain, bucket] of this.buckets) {
-            this.refill(domain, bucket);
-            result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };
-        }
-        return result;
+    const rps = this.rateMap[domain] ?? this.defaultRps;
+    const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1e3);
+    await new Promise((r) => setTimeout(r, waitMs));
+    bucket.tokens = 0;
+    bucket.lastRefill = Date.now();
+  }
+  /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
+  setRate(domain, rps) {
+    this.rateMap[domain] = rps;
+    const bucket = this.getBucket(domain);
+    bucket.tokens = 0;
+  }
+  stats() {
+    const result = {};
+    for (const [domain, bucket] of this.buckets) {
+      this.refill(domain, bucket);
+      result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };
     }
+    return result;
+  }
 }
+export {
+  DomainRateLimiter
+};
+//# sourceMappingURL=rate-limiter.js.map

package/dist/redirect-tracker.js

@@ -1,69 +1,65 @@
-/**
- * Redirect chain tracker — records the full redirect chain for crawled URLs.
- *
- * Why: redirect chains (A→B→C) slow down pages; circular redirects crash crawlers;
- * redirect loops indicate config errors. ZeTa records chains for SEO/perf audit.
- *
- * Integration: hook into Playwright's page.on('response') to see each redirect hop.
- */
-export async function trackRedirects(page, url) {
-    const hops = [];
-    const seen = new Set();
-    let lastResponseTime = Date.now();
-    const onResponse = (response) => {
-        const status = response.status();
-        if (status >= 300 && status < 400) {
-            const from = response.url();
-            const to = response.headers()['location'] ?? '';
-            const now = Date.now();
-            hops.push({ from, to, statusCode: status, durationMs: now - lastResponseTime });
-            lastResponseTime = now;
-        }
-    };
-    page.on('response', onResponse);
-    try {
-        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => { });
-    }
-    finally {
-        page.off('response', onResponse);
+async function trackRedirects(page, url) {
+  const hops = [];
+  const seen = /* @__PURE__ */ new Set();
+  let lastResponseTime = Date.now();
+  const onResponse = (response) => {
+    const status = response.status();
+    if (status >= 300 && status < 400) {
+      const from = response.url();
+      const to = response.headers()["location"] ?? "";
+      const now = Date.now();
+      hops.push({ from, to, statusCode: status, durationMs: now - lastResponseTime });
+      lastResponseTime = now;
     }
-    const finalUrl = page.url();
-    const isCircular = hops.some(h => {
-        if (seen.has(h.to))
-            return true;
-        seen.add(h.from);
-        return false;
+  };
+  page.on("response", onResponse);
+  try {
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch(() => {
     });
+  } finally {
+    page.off("response", onResponse);
+  }
+  const finalUrl = page.url();
+  const isCircular = hops.some((h) => {
+    if (seen.has(h.to)) return true;
+    seen.add(h.from);
+    return false;
+  });
+  return {
+    originalUrl: url,
+    finalUrl,
+    hops,
+    isCircular,
+    hopCount: hops.length,
+    totalDurationMs: hops.reduce((sum, h) => sum + h.durationMs, 0)
+  };
+}
+class RedirectRegistry {
+  chains = /* @__PURE__ */ new Map();
+  register(chain) {
+    this.chains.set(chain.originalUrl, chain);
+  }
+  get(url) {
+    return this.chains.get(url);
+  }
+  getAll() {
+    return [...this.chains.values()];
+  }
+  getProblematic() {
+    return [...this.chains.values()].filter((c) => c.isCircular || c.hopCount > 3);
+  }
+  stats() {
+    const all = [...this.chains.values()];
     return {
-        originalUrl: url,
-        finalUrl,
-        hops,
-        isCircular,
-        hopCount: hops.length,
-        totalDurationMs: hops.reduce((sum, h) => sum + h.durationMs, 0),
+      total: all.length,
+      withRedirects: all.filter((c) => c.hopCount > 0).length,
+      circular: all.filter((c) => c.isCircular).length,
+      deepChains: all.filter((c) => c.hopCount > 3).length
     };
+  }
 }
-export class RedirectRegistry {
-    chains = new Map();
-    register(chain) {
-        this.chains.set(chain.originalUrl, chain);
-    }
-    get(url) {
-        return this.chains.get(url);
-    }
-    getAll() {
-        return [...this.chains.values()];
-    }
-    getProblematic() {
-        return [...this.chains.values()].filter(c => c.isCircular || c.hopCount > 3);
-    }
-    stats() {
-        const all = [...this.chains.values()];
-        return {
-            total: all.length,
-            withRedirects: all.filter(c => c.hopCount > 0).length,
-            circular: all.filter(c => c.isCircular).length,
-            deepChains: all.filter(c => c.hopCount > 3).length,
-        };
-    }
-}
+export {
+  RedirectRegistry,
+  trackRedirects
+};
+//# sourceMappingURL=redirect-tracker.js.map