@detiq/crawler 0.1.3

package/src/proxy.ts

@@ -0,0 +1,238 @@
+/**
+ * Free proxy auto-selection for bot-protection bypass.
+ *
+ * Source: proxifly/free-proxy-list (MIT) via jsDelivr CDN.
+ * These are datacenter proxies — they bypass simple IP blocks but NOT
+ * full Cloudflare Bot Manager / Akamai. For Cloudflare-protected sites,
+ * users need a residential proxy (Bright Data, Oxylabs, etc.).
+ */
+
+import { createConnection } from 'net';
+
+const PROXY_LIST_URL =
+  'https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json';
+
+interface ProxyEntry {
+  ip: string;
+  port: number;
+  protocols: string[];
+  upTime?: number;
+  speed?: number;
+}
+
+/** Fetch and filter proxy list. Returns http proxies sorted by speed (fastest first). */
+async function fetchCandidates(): Promise<ProxyEntry[]> {
+  const res = await fetch(PROXY_LIST_URL, {
+    signal: AbortSignal.timeout(10_000),
+    headers: { 'Accept': 'application/json' },
+  });
+  if (!res.ok) throw new Error(`proxy list fetch failed: ${res.status}`);
+  const raw: ProxyEntry[] = await res.json();
+  return raw
+    .filter((p) => p.protocols?.includes('http') && p.port > 0 && (p.upTime ?? 0) >= 30)
+    .sort((a, b) => (a.speed ?? 99999) - (b.speed ?? 99999))
+    .slice(0, 50); // test at most 50 candidates
+}
+
+/** TCP-level proxy liveness check via HTTP CONNECT tunnel. No extra dependencies. */
+function testProxy(ip: string, port: number, timeoutMs: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    let settled = false;
+    const settle = (ok: boolean) => {
+      if (settled) return;
+      settled = true;
+      resolve(ok);
+      try { sock.destroy(); } catch { /* ignore */ }
+    };
+
+    const timer = setTimeout(() => settle(false), timeoutMs);
+    const sock = createConnection({ host: ip, port, timeout: timeoutMs });
+
+    sock.on('connect', () => {
+      sock.write('CONNECT example.com:80 HTTP/1.1\r\nHost: example.com:80\r\n\r\n');
+      sock.once('data', (data) => {
+        clearTimeout(timer);
+        const ok = /^HTTP\/1\.[01] 200/.test(data.toString());
+        settle(ok);
+      });
+    });
+    sock.on('error', () => { clearTimeout(timer); settle(false); });
+    sock.on('timeout', () => { clearTimeout(timer); settle(false); });
+  });
+}
+
+/** Test `batchSize` proxies concurrently and return first working one. */
+async function testBatch(
+  candidates: ProxyEntry[],
+  start: number,
+  batchSize: number,
+  timeoutMs: number,
+): Promise<string | null> {
+  const batch = candidates.slice(start, start + batchSize);
+  if (batch.length === 0) return null;
+
+  const results = await Promise.all(
+    batch.map(async (p) => {
+      const ok = await testProxy(p.ip, p.port, timeoutMs);
+      return ok ? `http://${p.ip}:${p.port}` : null;
+    }),
+  );
+
+  return results.find((r) => r !== null) ?? null;
+}
+
+/**
+ * Pick a working free proxy. Tests in parallel batches of 10 with a 4s timeout.
+ * Returns null if no working proxy found (not an error — crawl can proceed without proxy).
+ */
+export async function pickFreeProxy(): Promise<string | null> {
+  let candidates: ProxyEntry[];
+  try {
+    candidates = await fetchCandidates();
+  } catch (err: any) {
+    console.warn('[proxy] failed to fetch proxy list:', err?.message ?? err);
+    return null;
+  }
+
+  const BATCH = 10;
+  const TIMEOUT = 4_000;
+
+  for (let i = 0; i < candidates.length; i += BATCH) {
+    const found = await testBatch(candidates, i, BATCH, TIMEOUT);
+    if (found) {
+      console.info(`[proxy] selected free proxy: ${found} (tested ${i + BATCH} candidates)`);
+      return found;
+    }
+  }
+
+  console.warn('[proxy] no working free proxy found in list');
+  return null;
+}
+
+// ── Tiered Proxy Rotation ─────────────────────────────────────────────────────
+
+export type ProxyTier = 'free' | 'datacenter' | 'residential' | 'premium';
+
+export interface ProxyTierConfig {
+  /** List of proxy URLs for this tier. E.g. ['http://user:pass@dc1.proxy:8080'] */
+  urls: string[];
+  tier: ProxyTier;
+}
+
+interface DomainHistory {
+  tier: ProxyTier;
+  consecutiveFailures: number;
+  lastEscalatedAt: number;
+  totalRequests: number;
+  successRate: number;
+}
+
+const TIER_ORDER: ProxyTier[] = ['free', 'datacenter', 'residential', 'premium'];
+
+function nextTier(current: ProxyTier): ProxyTier {
+  const idx = TIER_ORDER.indexOf(current);
+  return TIER_ORDER[Math.min(idx + 1, TIER_ORDER.length - 1)];
+}
+
+/**
+ * Tiered proxy manager with per-domain error histograms.
+ *
+ * Tracks success/failure per domain and automatically escalates to higher proxy tiers
+ * on consecutive failures. Applies time-decay to probe returning to lower tiers.
+ *
+ * Usage:
+ *   const mgr = new TieredProxyManager([{ tier: 'datacenter', urls: [...] }]);
+ *   const proxy = await mgr.selectProxy('api.salesforce.com');
+ *   // ... make request ...
+ *   mgr.recordResult('api.salesforce.com', success);
+ */
+export class TieredProxyManager {
+  private tiers: Map<ProxyTier, string[]> = new Map();
+  private domainHistory: Map<string, DomainHistory> = new Map();
+  private readonly ESCALATE_AFTER = 3; // consecutive failures before escalation
+  private readonly DECAY_AFTER_MS = 30 * 60 * 1000; // 30 min: probe tier downgrade
+
+  constructor(configs: ProxyTierConfig[]) {
+    for (const cfg of configs) {
+      if (cfg.urls.length > 0) this.tiers.set(cfg.tier, cfg.urls);
+    }
+  }
+
+  private getHistory(domain: string): DomainHistory {
+    if (!this.domainHistory.has(domain)) {
+      this.domainHistory.set(domain, {
+        tier: this.lowestAvailableTier(),
+        consecutiveFailures: 0,
+        lastEscalatedAt: 0,
+        totalRequests: 0,
+        successRate: 1,
+      });
+    }
+    return this.domainHistory.get(domain)!;
+  }
+
+  private lowestAvailableTier(): ProxyTier {
+    for (const t of TIER_ORDER) {
+      if (this.tiers.has(t)) return t;
+    }
+    return 'free';
+  }
+
+  /** Pick a proxy URL for this domain, based on its tier history. */
+  selectProxy(domain: string): string | null {
+    const history = this.getHistory(domain);
+
+    // Time-decay: if we've been on elevated tier for DECAY_AFTER_MS, try stepping down
+    if (history.consecutiveFailures === 0 && history.lastEscalatedAt > 0) {
+      const elapsed = Date.now() - history.lastEscalatedAt;
+      if (elapsed > this.DECAY_AFTER_MS) {
+        const currentIdx = TIER_ORDER.indexOf(history.tier);
+        if (currentIdx > 0) {
+          const lowerTier = TIER_ORDER[currentIdx - 1];
+          if (this.tiers.has(lowerTier)) {
+            history.tier = lowerTier;
+            history.lastEscalatedAt = 0;
+          }
+        }
+      }
+    }
+
+    const urls = this.tiers.get(history.tier);
+    if (!urls || urls.length === 0) return null;
+    // Round-robin within tier
+    const idx = history.totalRequests % urls.length;
+    return urls[idx];
+  }
+
+  /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
+  recordResult(domain: string, success: boolean): void {
+    const history = this.getHistory(domain);
+    history.totalRequests++;
+    if (success) {
+      history.consecutiveFailures = 0;
+      // Update EMA success rate
+      history.successRate = history.successRate * 0.9 + 0.1;
+    } else {
+      history.consecutiveFailures++;
+      history.successRate = history.successRate * 0.9;
+      if (history.consecutiveFailures >= this.ESCALATE_AFTER) {
+        const elevated = nextTier(history.tier);
+        if (elevated !== history.tier && this.tiers.has(elevated)) {
+          console.warn(`[proxy-tier] domain=${domain} escalating ${history.tier}→${elevated} after ${history.consecutiveFailures} failures`);
+          history.tier = elevated;
+          history.lastEscalatedAt = Date.now();
+          history.consecutiveFailures = 0;
+        }
+      }
+    }
+  }
+
+  stats(): Array<{ domain: string; tier: ProxyTier; successRate: number; totalRequests: number }> {
+    return Array.from(this.domainHistory.entries()).map(([domain, h]) => ({
+      domain,
+      tier: h.tier,
+      successRate: Math.round(h.successRate * 100) / 100,
+      totalRequests: h.totalRequests,
+    }));
+  }
+}

package/src/rate-limiter.ts

@@ -0,0 +1,87 @@
+/**
+ * Domain-aware rate limiter — ensures the crawler respects per-domain rate limits.
+ *
+ * Uses a token bucket algorithm per domain. When the bucket is empty, requests
+ * wait until a token is available. Tokens refill at a configurable rate.
+ *
+ * Default: 2 requests/second per domain, burst of 5.
+ * Can be configured per-domain via the rateMap (domain → requests/second).
+ */
+
+export interface RateLimiterOpts {
+  defaultRps?: number;        // requests per second (default 2)
+  burstSize?: number;         // max burst (default 5)
+  rateMap?: Record<string, number>; // domain-specific overrides
+}
+
+interface Bucket {
+  tokens: number;
+  lastRefill: number; // Date.now()
+}
+
+function extractDomain(url: string): string {
+  try { return new URL(url).hostname; } catch { return url; }
+}
+
+export class DomainRateLimiter {
+  private buckets = new Map<string, Bucket>();
+  private readonly defaultRps: number;
+  private readonly burstSize: number;
+  private readonly rateMap: Record<string, number>;
+
+  constructor(opts: RateLimiterOpts = {}) {
+    this.defaultRps = opts.defaultRps ?? 2;
+    this.burstSize = opts.burstSize ?? 5;
+    this.rateMap = opts.rateMap ?? {};
+  }
+
+  private getBucket(domain: string): Bucket {
+    if (!this.buckets.has(domain)) {
+      this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });
+    }
+    return this.buckets.get(domain)!;
+  }
+
+  private refill(domain: string, bucket: Bucket): void {
+    const rps = this.rateMap[domain] ?? this.defaultRps;
+    const now = Date.now();
+    const elapsed = (now - bucket.lastRefill) / 1000;
+    bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);
+    bucket.lastRefill = now;
+  }
+
+  /** Wait until a token is available for the given URL's domain. */
+  async throttle(url: string): Promise<void> {
+    const domain = extractDomain(url);
+    const bucket = this.getBucket(domain);
+    this.refill(domain, bucket);
+
+    if (bucket.tokens >= 1) {
+      bucket.tokens -= 1;
+      return;
+    }
+
+    // Calculate wait time until next token
+    const rps = this.rateMap[domain] ?? this.defaultRps;
+    const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1000);
+    await new Promise(r => setTimeout(r, waitMs));
+    bucket.tokens = 0;
+    bucket.lastRefill = Date.now();
+  }
+
+  /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
+  setRate(domain: string, rps: number): void {
+    this.rateMap[domain] = rps;
+    const bucket = this.getBucket(domain);
+    bucket.tokens = 0; // drain immediately on rate change
+  }
+
+  stats(): Record<string, { tokens: number; rps: number }> {
+    const result: Record<string, any> = {};
+    for (const [domain, bucket] of this.buckets) {
+      this.refill(domain, bucket);
+      result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };
+    }
+    return result;
+  }
+}

package/src/redirect-tracker.ts

@@ -0,0 +1,97 @@
+/**
+ * Redirect chain tracker — records the full redirect chain for crawled URLs.
+ *
+ * Why: redirect chains (A→B→C) slow down pages; circular redirects crash crawlers;
+ * redirect loops indicate config errors. ZeTa records chains for SEO/perf audit.
+ *
+ * Integration: hook into Playwright's page.on('response') to see each redirect hop.
+ */
+
+export interface RedirectHop {
+  from: string;
+  to: string;
+  statusCode: number;
+  durationMs: number;
+}
+
+export interface RedirectChain {
+  originalUrl: string;
+  finalUrl: string;
+  hops: RedirectHop[];
+  isCircular: boolean;
+  hopCount: number;
+  totalDurationMs: number;
+}
+
+import type { Page } from 'playwright';
+
+export async function trackRedirects(page: Page, url: string): Promise<RedirectChain> {
+  const hops: RedirectHop[] = [];
+  const seen = new Set<string>();
+  let lastResponseTime = Date.now();
+
+  const onResponse = (response: any) => {
+    const status = response.status();
+    if (status >= 300 && status < 400) {
+      const from = response.url();
+      const to = response.headers()['location'] ?? '';
+      const now = Date.now();
+      hops.push({ from, to, statusCode: status, durationMs: now - lastResponseTime });
+      lastResponseTime = now;
+    }
+  };
+
+  page.on('response', onResponse);
+
+  try {
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});
+  } finally {
+    page.off('response', onResponse);
+  }
+
+  const finalUrl = page.url();
+  const isCircular = hops.some(h => {
+    if (seen.has(h.to)) return true;
+    seen.add(h.from);
+    return false;
+  });
+
+  return {
+    originalUrl: url,
+    finalUrl,
+    hops,
+    isCircular,
+    hopCount: hops.length,
+    totalDurationMs: hops.reduce((sum, h) => sum + h.durationMs, 0),
+  };
+}
+
+export class RedirectRegistry {
+  private chains = new Map<string, RedirectChain>();
+
+  register(chain: RedirectChain): void {
+    this.chains.set(chain.originalUrl, chain);
+  }
+
+  get(url: string): RedirectChain | undefined {
+    return this.chains.get(url);
+  }
+
+  getAll(): RedirectChain[] {
+    return [...this.chains.values()];
+  }
+
+  getProblematic(): RedirectChain[] {
+    return [...this.chains.values()].filter(c => c.isCircular || c.hopCount > 3);
+  }
+
+  stats(): { total: number; withRedirects: number; circular: number; deepChains: number } {
+    const all = [...this.chains.values()];
+    return {
+      total: all.length,
+      withRedirects: all.filter(c => c.hopCount > 0).length,
+      circular: all.filter(c => c.isCircular).length,
+      deepChains: all.filter(c => c.hopCount > 3).length,
+    };
+  }
+}

package/src/redis-frontier.ts

@@ -0,0 +1,143 @@
+/**
+ * Redis-backed distributed crawl frontier.
+ *
+ * Replaces the in-memory CrawlFrontier for horizontally-scaled crawler deployments.
+ * Multiple crawler workers on different containers can share the same URL queue for
+ * a given job — enabling 10x crawl throughput on large enterprise apps.
+ *
+ * Redis key namespace:  zeta:frontier:{jobId}
+ *   LIST  zeta:frontier:{jobId}:queue   — BFS queue (LPUSH/RPOP)
+ *   SET   zeta:frontier:{jobId}:visited — deduplication set
+ *   EXPIRE                               — 24h TTL on all keys
+ *
+ * Graceful fallback: if REDIS_URL is not set, RedisCrawlFrontier silently falls
+ * back to in-memory arrays so local development works without Redis.
+ */
+
+const FRONTIER_TTL = 86400; // 24h in seconds
+
+export class RedisCrawlFrontier {
+  private redis: any;
+  private queueKey: string;
+  private visitedKey: string;
+  private inMemoryQueue: string[] = [];
+  private inMemoryVisited = new Set<string>();
+  private useRedis: boolean;
+
+  constructor(private readonly jobId: string) {
+    this.queueKey = `zeta:frontier:${jobId}:queue`;
+    this.visitedKey = `zeta:frontier:${jobId}:visited`;
+    this.useRedis = !!process.env.REDIS_URL;
+  }
+
+  async init(): Promise<void> {
+    if (!this.useRedis) return;
+    try {
+      const Redis: any = await import('ioredis').then(m => m.default ?? m);
+      this.redis = new Redis(process.env.REDIS_URL!, {
+        maxRetriesPerRequest: 2,
+        enableReadyCheck: false,
+        lazyConnect: true,
+      });
+      await this.redis.connect().catch(() => {
+        this.useRedis = false;
+        this.redis = null;
+      });
+    } catch {
+      this.useRedis = false;
+      this.redis = null;
+    }
+  }
+
+  /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
+  async push(url: string): Promise<boolean> {
+    if (this.useRedis && this.redis) {
+      try {
+        const isNew = await this.redis.sadd(this.visitedKey, url);
+        if (!isNew) return false;
+        await this.redis.lpush(this.queueKey, url);
+        await this.redis.expire(this.queueKey, FRONTIER_TTL);
+        await this.redis.expire(this.visitedKey, FRONTIER_TTL);
+        return true;
+      } catch {
+        // Fall through to in-memory on Redis error
+      }
+    }
+    if (this.inMemoryVisited.has(url)) return false;
+    this.inMemoryVisited.add(url);
+    this.inMemoryQueue.push(url);
+    return true;
+  }
+
+  /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
+  async shift(): Promise<string | null> {
+    if (this.useRedis && this.redis) {
+      try {
+        const url = await this.redis.rpop(this.queueKey);
+        return url ?? null;
+      } catch {
+        // Fall through
+      }
+    }
+    return this.inMemoryQueue.shift() ?? null;
+  }
+
+  /** Check if a URL has already been visited/queued. */
+  async hasVisited(url: string): Promise<boolean> {
+    if (this.useRedis && this.redis) {
+      try {
+        return (await this.redis.sismember(this.visitedKey, url)) === 1;
+      } catch { /* fall through */ }
+    }
+    return this.inMemoryVisited.has(url);
+  }
+
+  /** Current queue length. */
+  async length(): Promise<number> {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.llen(this.queueKey);
+      } catch { /* fall through */ }
+    }
+    return this.inMemoryQueue.length;
+  }
+
+  /** Total visited count. */
+  async visitedCount(): Promise<number> {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.scard(this.visitedKey);
+      } catch { /* fall through */ }
+    }
+    return this.inMemoryVisited.size;
+  }
+
+  /** Seed initial URLs (used at job start). */
+  async seed(urls: string[]): Promise<void> {
+    for (const url of urls) await this.push(url);
+  }
+
+  /** Clean up all Redis keys for this frontier. Call at job completion. */
+  async cleanup(): Promise<void> {
+    if (this.useRedis && this.redis) {
+      try {
+        await this.redis.del(this.queueKey, this.visitedKey);
+      } catch { /* ignore */ }
+      try { await this.redis.quit(); } catch { /* ignore */ }
+    }
+  }
+
+  /** Stats for monitoring. */
+  async stats(): Promise<{ backend: 'redis' | 'memory'; queued: number; visited: number }> {
+    return {
+      backend: this.useRedis && this.redis ? 'redis' : 'memory',
+      queued: await this.length(),
+      visited: await this.visitedCount(),
+    };
+  }
+}
+
+/** Check if Redis is configured for distributed frontier. */
+export function isRedisFrontierEnabled(): boolean {
+  return !!process.env.REDIS_URL;
+}