@detiq/crawler 1.1.8 → 1.1.9

package/dist/proxy.js

@@ -1,165 +1,191 @@
-import { createConnection } from "net";
-const PROXY_LIST_URL = "https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json";
+/**
+ * Free proxy auto-selection for bot-protection bypass.
+ *
+ * Source: proxifly/free-proxy-list (MIT) via jsDelivr CDN.
+ * These are datacenter proxies — they bypass simple IP blocks but NOT
+ * full Cloudflare Bot Manager / Akamai. For Cloudflare-protected sites,
+ * users need a residential proxy (Bright Data, Oxylabs, etc.).
+ */
+import { createConnection } from 'net';
+const PROXY_LIST_URL = 'https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json';
+/** Fetch and filter proxy list. Returns http proxies sorted by speed (fastest first). */
 async function fetchCandidates() {
-  const res = await fetch(PROXY_LIST_URL, {
-    signal: AbortSignal.timeout(1e4),
-    headers: { "Accept": "application/json" }
-  });
-  if (!res.ok) throw new Error(`proxy list fetch failed: ${res.status}`);
-  const raw = await res.json();
-  return raw.filter((p) => p.protocols?.includes("http") && p.port > 0 && (p.upTime ?? 0) >= 30).sort((a, b) => (a.speed ?? 99999) - (b.speed ?? 99999)).slice(0, 50);
+    const res = await fetch(PROXY_LIST_URL, {
+        signal: AbortSignal.timeout(10_000),
+        headers: { 'Accept': 'application/json' },
+    });
+    if (!res.ok)
+        throw new Error(`proxy list fetch failed: ${res.status}`);
+    const raw = await res.json();
+    return raw
+        .filter((p) => p.protocols?.includes('http') && p.port > 0 && (p.upTime ?? 0) >= 30)
+        .sort((a, b) => (a.speed ?? 99999) - (b.speed ?? 99999))
+        .slice(0, 50); // test at most 50 candidates
 }
+/** TCP-level proxy liveness check via HTTP CONNECT tunnel. No extra dependencies. */
 function testProxy(ip, port, timeoutMs) {
-  return new Promise((resolve) => {
-    let settled = false;
-    const settle = (ok) => {
-      if (settled) return;
-      settled = true;
-      resolve(ok);
-      try {
-        sock.destroy();
-      } catch {
-      }
-    };
-    const timer = setTimeout(() => settle(false), timeoutMs);
-    const sock = createConnection({ host: ip, port, timeout: timeoutMs });
-    sock.on("connect", () => {
-      sock.write("CONNECT example.com:80 HTTP/1.1\r\nHost: example.com:80\r\n\r\n");
-      sock.once("data", (data) => {
-        clearTimeout(timer);
-        const ok = /^HTTP\/1\.[01] 200/.test(data.toString());
-        settle(ok);
-      });
-    });
-    sock.on("error", () => {
-      clearTimeout(timer);
-      settle(false);
+    return new Promise((resolve) => {
+        let settled = false;
+        const settle = (ok) => {
+            if (settled)
+                return;
+            settled = true;
+            resolve(ok);
+            try {
+                sock.destroy();
+            }
+            catch { /* ignore */ }
+        };
+        const timer = setTimeout(() => settle(false), timeoutMs);
+        const sock = createConnection({ host: ip, port, timeout: timeoutMs });
+        sock.on('connect', () => {
+            sock.write('CONNECT example.com:80 HTTP/1.1\r\nHost: example.com:80\r\n\r\n');
+            sock.once('data', (data) => {
+                clearTimeout(timer);
+                const ok = /^HTTP\/1\.[01] 200/.test(data.toString());
+                settle(ok);
+            });
+        });
+        sock.on('error', () => { clearTimeout(timer); settle(false); });
+        sock.on('timeout', () => { clearTimeout(timer); settle(false); });
     });
-    sock.on("timeout", () => {
-      clearTimeout(timer);
-      settle(false);
-    });
-  });
 }
+/** Test `batchSize` proxies concurrently and return first working one. */
 async function testBatch(candidates, start, batchSize, timeoutMs) {
-  const batch = candidates.slice(start, start + batchSize);
-  if (batch.length === 0) return null;
-  const results = await Promise.all(
-    batch.map(async (p) => {
-      const ok = await testProxy(p.ip, p.port, timeoutMs);
-      return ok ? `http://${p.ip}:${p.port}` : null;
-    })
-  );
-  return results.find((r) => r !== null) ?? null;
+    const batch = candidates.slice(start, start + batchSize);
+    if (batch.length === 0)
+        return null;
+    const results = await Promise.all(batch.map(async (p) => {
+        const ok = await testProxy(p.ip, p.port, timeoutMs);
+        return ok ? `http://${p.ip}:${p.port}` : null;
+    }));
+    return results.find((r) => r !== null) ?? null;
 }
-async function pickFreeProxy() {
-  let candidates;
-  try {
-    candidates = await fetchCandidates();
-  } catch (err) {
-    console.warn("[proxy] failed to fetch proxy list:", err?.message ?? err);
-    return null;
-  }
-  const BATCH = 10;
-  const TIMEOUT = 4e3;
-  for (let i = 0; i < candidates.length; i += BATCH) {
-    const found = await testBatch(candidates, i, BATCH, TIMEOUT);
-    if (found) {
-      console.info(`[proxy] selected free proxy: ${found} (tested ${i + BATCH} candidates)`);
-      return found;
+/**
+ * Pick a working free proxy. Tests in parallel batches of 10 with a 4s timeout.
+ * Returns null if no working proxy found (not an error — crawl can proceed without proxy).
+ */
+export async function pickFreeProxy() {
+    let candidates;
+    try {
+        candidates = await fetchCandidates();
+    }
+    catch (err) {
+        console.warn('[proxy] failed to fetch proxy list:', err?.message ?? err);
+        return null;
+    }
+    const BATCH = 10;
+    const TIMEOUT = 4_000;
+    for (let i = 0; i < candidates.length; i += BATCH) {
+        const found = await testBatch(candidates, i, BATCH, TIMEOUT);
+        if (found) {
+            console.info(`[proxy] selected free proxy: ${found} (tested ${i + BATCH} candidates)`);
+            return found;
+        }
     }
-  }
-  console.warn("[proxy] no working free proxy found in list");
-  return null;
+    console.warn('[proxy] no working free proxy found in list');
+    return null;
 }
-const TIER_ORDER = ["free", "datacenter", "residential", "premium"];
+const TIER_ORDER = ['free', 'datacenter', 'residential', 'premium'];
 function nextTier(current) {
-  const idx = TIER_ORDER.indexOf(current);
-  return TIER_ORDER[Math.min(idx + 1, TIER_ORDER.length - 1)];
+    const idx = TIER_ORDER.indexOf(current);
+    return TIER_ORDER[Math.min(idx + 1, TIER_ORDER.length - 1)];
 }
-class TieredProxyManager {
-  tiers = /* @__PURE__ */ new Map();
-  domainHistory = /* @__PURE__ */ new Map();
-  ESCALATE_AFTER = 3;
-  // consecutive failures before escalation
-  DECAY_AFTER_MS = 30 * 60 * 1e3;
-  // 30 min: probe tier downgrade
-  constructor(configs) {
-    for (const cfg of configs) {
-      if (cfg.urls.length > 0) this.tiers.set(cfg.tier, cfg.urls);
+/**
+ * Tiered proxy manager with per-domain error histograms.
+ *
+ * Tracks success/failure per domain and automatically escalates to higher proxy tiers
+ * on consecutive failures. Applies time-decay to probe returning to lower tiers.
+ *
+ * Usage:
+ *   const mgr = new TieredProxyManager([{ tier: 'datacenter', urls: [...] }]);
+ *   const proxy = await mgr.selectProxy('api.salesforce.com');
+ *   // ... make request ...
+ *   mgr.recordResult('api.salesforce.com', success);
+ */
+export class TieredProxyManager {
+    tiers = new Map();
+    domainHistory = new Map();
+    ESCALATE_AFTER = 3; // consecutive failures before escalation
+    DECAY_AFTER_MS = 30 * 60 * 1000; // 30 min: probe tier downgrade
+    constructor(configs) {
+        for (const cfg of configs) {
+            if (cfg.urls.length > 0)
+                this.tiers.set(cfg.tier, cfg.urls);
+        }
     }
-  }
-  getHistory(domain) {
-    if (!this.domainHistory.has(domain)) {
-      this.domainHistory.set(domain, {
-        tier: this.lowestAvailableTier(),
-        consecutiveFailures: 0,
-        lastEscalatedAt: 0,
-        totalRequests: 0,
-        successRate: 1
-      });
+    getHistory(domain) {
+        if (!this.domainHistory.has(domain)) {
+            this.domainHistory.set(domain, {
+                tier: this.lowestAvailableTier(),
+                consecutiveFailures: 0,
+                lastEscalatedAt: 0,
+                totalRequests: 0,
+                successRate: 1,
+            });
+        }
+        return this.domainHistory.get(domain);
     }
-    return this.domainHistory.get(domain);
-  }
-  lowestAvailableTier() {
-    for (const t of TIER_ORDER) {
-      if (this.tiers.has(t)) return t;
+    lowestAvailableTier() {
+        for (const t of TIER_ORDER) {
+            if (this.tiers.has(t))
+                return t;
+        }
+        return 'free';
     }
-    return "free";
-  }
-  /** Pick a proxy URL for this domain, based on its tier history. */
-  selectProxy(domain) {
-    const history = this.getHistory(domain);
-    if (history.consecutiveFailures === 0 && history.lastEscalatedAt > 0) {
-      const elapsed = Date.now() - history.lastEscalatedAt;
-      if (elapsed > this.DECAY_AFTER_MS) {
-        const currentIdx = TIER_ORDER.indexOf(history.tier);
-        if (currentIdx > 0) {
-          const lowerTier = TIER_ORDER[currentIdx - 1];
-          if (this.tiers.has(lowerTier)) {
-            history.tier = lowerTier;
-            history.lastEscalatedAt = 0;
-          }
+    /** Pick a proxy URL for this domain, based on its tier history. */
+    selectProxy(domain) {
+        const history = this.getHistory(domain);
+        // Time-decay: if we've been on elevated tier for DECAY_AFTER_MS, try stepping down
+        if (history.consecutiveFailures === 0 && history.lastEscalatedAt > 0) {
+            const elapsed = Date.now() - history.lastEscalatedAt;
+            if (elapsed > this.DECAY_AFTER_MS) {
+                const currentIdx = TIER_ORDER.indexOf(history.tier);
+                if (currentIdx > 0) {
+                    const lowerTier = TIER_ORDER[currentIdx - 1];
+                    if (this.tiers.has(lowerTier)) {
+                        history.tier = lowerTier;
+                        history.lastEscalatedAt = 0;
+                    }
+                }
+            }
         }
-      }
+        const urls = this.tiers.get(history.tier);
+        if (!urls || urls.length === 0)
+            return null;
+        // Round-robin within tier
+        const idx = history.totalRequests % urls.length;
+        return urls[idx];
     }
-    const urls = this.tiers.get(history.tier);
-    if (!urls || urls.length === 0) return null;
-    const idx = history.totalRequests % urls.length;
-    return urls[idx];
-  }
-  /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
-  recordResult(domain, success) {
-    const history = this.getHistory(domain);
-    history.totalRequests++;
-    if (success) {
-      history.consecutiveFailures = 0;
-      history.successRate = history.successRate * 0.9 + 0.1;
-    } else {
-      history.consecutiveFailures++;
-      history.successRate = history.successRate * 0.9;
-      if (history.consecutiveFailures >= this.ESCALATE_AFTER) {
-        const elevated = nextTier(history.tier);
-        if (elevated !== history.tier && this.tiers.has(elevated)) {
-          console.warn(`[proxy-tier] domain=${domain} escalating ${history.tier}\u2192${elevated} after ${history.consecutiveFailures} failures`);
-          history.tier = elevated;
-          history.lastEscalatedAt = Date.now();
-          history.consecutiveFailures = 0;
+    /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
+    recordResult(domain, success) {
+        const history = this.getHistory(domain);
+        history.totalRequests++;
+        if (success) {
+            history.consecutiveFailures = 0;
+            // Update EMA success rate
+            history.successRate = history.successRate * 0.9 + 0.1;
+        }
+        else {
+            history.consecutiveFailures++;
+            history.successRate = history.successRate * 0.9;
+            if (history.consecutiveFailures >= this.ESCALATE_AFTER) {
+                const elevated = nextTier(history.tier);
+                if (elevated !== history.tier && this.tiers.has(elevated)) {
+                    console.warn(`[proxy-tier] domain=${domain} escalating ${history.tier}→${elevated} after ${history.consecutiveFailures} failures`);
+                    history.tier = elevated;
+                    history.lastEscalatedAt = Date.now();
+                    history.consecutiveFailures = 0;
+                }
+            }
         }
-      }
     }
-  }
-  stats() {
-    return Array.from(this.domainHistory.entries()).map(([domain, h]) => ({
-      domain,
-      tier: h.tier,
-      successRate: Math.round(h.successRate * 100) / 100,
-      totalRequests: h.totalRequests
-    }));
-  }
+    stats() {
+        return Array.from(this.domainHistory.entries()).map(([domain, h]) => ({
+            domain,
+            tier: h.tier,
+            successRate: Math.round(h.successRate * 100) / 100,
+            totalRequests: h.totalRequests,
+        }));
+    }
 }
-export {
-  TieredProxyManager,
-  pickFreeProxy
-};
-//# sourceMappingURL=proxy.js.map

package/dist/rate-limiter.js

@@ -1,64 +1,71 @@
+/**
+ * Domain-aware rate limiter — ensures the crawler respects per-domain rate limits.
+ *
+ * Uses a token bucket algorithm per domain. When the bucket is empty, requests
+ * wait until a token is available. Tokens refill at a configurable rate.
+ *
+ * Default: 2 requests/second per domain, burst of 5.
+ * Can be configured per-domain via the rateMap (domain → requests/second).
+ */
 function extractDomain(url) {
-  try {
-    return new URL(url).hostname;
-  } catch {
-    return url;
-  }
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return url;
+    }
 }
-class DomainRateLimiter {
-  buckets = /* @__PURE__ */ new Map();
-  defaultRps;
-  burstSize;
-  rateMap;
-  constructor(opts = {}) {
-    this.defaultRps = opts.defaultRps ?? 2;
-    this.burstSize = opts.burstSize ?? 5;
-    this.rateMap = opts.rateMap ?? {};
-  }
-  getBucket(domain) {
-    if (!this.buckets.has(domain)) {
-      this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });
+export class DomainRateLimiter {
+    buckets = new Map();
+    defaultRps;
+    burstSize;
+    rateMap;
+    constructor(opts = {}) {
+        this.defaultRps = opts.defaultRps ?? 2;
+        this.burstSize = opts.burstSize ?? 5;
+        this.rateMap = opts.rateMap ?? {};
+    }
+    getBucket(domain) {
+        if (!this.buckets.has(domain)) {
+            this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });
+        }
+        return this.buckets.get(domain);
+    }
+    refill(domain, bucket) {
+        const rps = this.rateMap[domain] ?? this.defaultRps;
+        const now = Date.now();
+        const elapsed = (now - bucket.lastRefill) / 1000;
+        bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);
+        bucket.lastRefill = now;
+    }
+    /** Wait until a token is available for the given URL's domain. */
+    async throttle(url) {
+        const domain = extractDomain(url);
+        const bucket = this.getBucket(domain);
+        this.refill(domain, bucket);
+        if (bucket.tokens >= 1) {
+            bucket.tokens -= 1;
+            return;
+        }
+        // Calculate wait time until next token
+        const rps = this.rateMap[domain] ?? this.defaultRps;
+        const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1000);
+        await new Promise(r => setTimeout(r, waitMs));
+        bucket.tokens = 0;
+        bucket.lastRefill = Date.now();
     }
-    return this.buckets.get(domain);
-  }
-  refill(domain, bucket) {
-    const rps = this.rateMap[domain] ?? this.defaultRps;
-    const now = Date.now();
-    const elapsed = (now - bucket.lastRefill) / 1e3;
-    bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);
-    bucket.lastRefill = now;
-  }
-  /** Wait until a token is available for the given URL's domain. */
-  async throttle(url) {
-    const domain = extractDomain(url);
-    const bucket = this.getBucket(domain);
-    this.refill(domain, bucket);
-    if (bucket.tokens >= 1) {
-      bucket.tokens -= 1;
-      return;
+    /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
+    setRate(domain, rps) {
+        this.rateMap[domain] = rps;
+        const bucket = this.getBucket(domain);
+        bucket.tokens = 0; // drain immediately on rate change
     }
-    const rps = this.rateMap[domain] ?? this.defaultRps;
-    const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1e3);
-    await new Promise((r) => setTimeout(r, waitMs));
-    bucket.tokens = 0;
-    bucket.lastRefill = Date.now();
-  }
-  /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
-  setRate(domain, rps) {
-    this.rateMap[domain] = rps;
-    const bucket = this.getBucket(domain);
-    bucket.tokens = 0;
-  }
-  stats() {
-    const result = {};
-    for (const [domain, bucket] of this.buckets) {
-      this.refill(domain, bucket);
-      result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };
+    stats() {
+        const result = {};
+        for (const [domain, bucket] of this.buckets) {
+            this.refill(domain, bucket);
+            result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };
+        }
+        return result;
     }
-    return result;
-  }
 }
-export {
-  DomainRateLimiter
-};
-//# sourceMappingURL=rate-limiter.js.map

package/dist/redirect-tracker.js

@@ -1,65 +1,69 @@
-async function trackRedirects(page, url) {
-  const hops = [];
-  const seen = /* @__PURE__ */ new Set();
-  let lastResponseTime = Date.now();
-  const onResponse = (response) => {
-    const status = response.status();
-    if (status >= 300 && status < 400) {
-      const from = response.url();
-      const to = response.headers()["location"] ?? "";
-      const now = Date.now();
-      hops.push({ from, to, statusCode: status, durationMs: now - lastResponseTime });
-      lastResponseTime = now;
+/**
+ * Redirect chain tracker — records the full redirect chain for crawled URLs.
+ *
+ * Why: redirect chains (A→B→C) slow down pages; circular redirects crash crawlers;
+ * redirect loops indicate config errors. ZeTa records chains for SEO/perf audit.
+ *
+ * Integration: hook into Playwright's page.on('response') to see each redirect hop.
+ */
+export async function trackRedirects(page, url) {
+    const hops = [];
+    const seen = new Set();
+    let lastResponseTime = Date.now();
+    const onResponse = (response) => {
+        const status = response.status();
+        if (status >= 300 && status < 400) {
+            const from = response.url();
+            const to = response.headers()['location'] ?? '';
+            const now = Date.now();
+            hops.push({ from, to, statusCode: status, durationMs: now - lastResponseTime });
+            lastResponseTime = now;
+        }
+    };
+    page.on('response', onResponse);
+    try {
+        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => { });
+    }
+    finally {
+        page.off('response', onResponse);
     }
-  };
-  page.on("response", onResponse);
-  try {
-    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 3e4 }).catch(() => {
+    const finalUrl = page.url();
+    const isCircular = hops.some(h => {
+        if (seen.has(h.to))
+            return true;
+        seen.add(h.from);
+        return false;
     });
-  } finally {
-    page.off("response", onResponse);
-  }
-  const finalUrl = page.url();
-  const isCircular = hops.some((h) => {
-    if (seen.has(h.to)) return true;
-    seen.add(h.from);
-    return false;
-  });
-  return {
-    originalUrl: url,
-    finalUrl,
-    hops,
-    isCircular,
-    hopCount: hops.length,
-    totalDurationMs: hops.reduce((sum, h) => sum + h.durationMs, 0)
-  };
-}
-class RedirectRegistry {
-  chains = /* @__PURE__ */ new Map();
-  register(chain) {
-    this.chains.set(chain.originalUrl, chain);
-  }
-  get(url) {
-    return this.chains.get(url);
-  }
-  getAll() {
-    return [...this.chains.values()];
-  }
-  getProblematic() {
-    return [...this.chains.values()].filter((c) => c.isCircular || c.hopCount > 3);
-  }
-  stats() {
-    const all = [...this.chains.values()];
     return {
-      total: all.length,
-      withRedirects: all.filter((c) => c.hopCount > 0).length,
-      circular: all.filter((c) => c.isCircular).length,
-      deepChains: all.filter((c) => c.hopCount > 3).length
+        originalUrl: url,
+        finalUrl,
+        hops,
+        isCircular,
+        hopCount: hops.length,
+        totalDurationMs: hops.reduce((sum, h) => sum + h.durationMs, 0),
     };
-  }
 }
-export {
-  RedirectRegistry,
-  trackRedirects
-};
-//# sourceMappingURL=redirect-tracker.js.map
+export class RedirectRegistry {
+    chains = new Map();
+    register(chain) {
+        this.chains.set(chain.originalUrl, chain);
+    }
+    get(url) {
+        return this.chains.get(url);
+    }
+    getAll() {
+        return [...this.chains.values()];
+    }
+    getProblematic() {
+        return [...this.chains.values()].filter(c => c.isCircular || c.hopCount > 3);
+    }
+    stats() {
+        const all = [...this.chains.values()];
+        return {
+            total: all.length,
+            withRedirects: all.filter(c => c.hopCount > 0).length,
+            circular: all.filter(c => c.isCircular).length,
+            deepChains: all.filter(c => c.hopCount > 3).length,
+        };
+    }
+}