@taodigital/crawler 0.0.0-stage → 1.2.1

package/src/rate-limiter.ts

@@ -0,0 +1,87 @@
+/**
+ * Domain-aware rate limiter — ensures the crawler respects per-domain rate limits.
+ *
+ * Uses a token bucket algorithm per domain. When the bucket is empty, requests
+ * wait until a token is available. Tokens refill at a configurable rate.
+ *
+ * Default: 2 requests/second per domain, burst of 5.
+ * Can be configured per-domain via the rateMap (domain → requests/second).
+ */
+
+export interface RateLimiterOpts {
+  defaultRps?: number;        // requests per second (default 2)
+  burstSize?: number;         // max burst (default 5)
+  rateMap?: Record<string, number>; // domain-specific overrides
+}
+
+interface Bucket {
+  tokens: number;
+  lastRefill: number; // Date.now()
+}
+
+function extractDomain(url: string): string {
+  try { return new URL(url).hostname; } catch { return url; }
+}
+
+export class DomainRateLimiter {
+  private buckets = new Map<string, Bucket>();
+  private readonly defaultRps: number;
+  private readonly burstSize: number;
+  private readonly rateMap: Record<string, number>;
+
+  constructor(opts: RateLimiterOpts = {}) {
+    this.defaultRps = opts.defaultRps ?? (parseFloat(process.env.CRAWLER_RATE_LIMIT_DEFAULT_RPS ?? '') || 2);
+    this.burstSize = opts.burstSize ?? (parseInt(process.env.CRAWLER_RATE_LIMIT_BURST_SIZE ?? '') || 5);
+    this.rateMap = opts.rateMap ?? {};
+  }
+
+  private getBucket(domain: string): Bucket {
+    if (!this.buckets.has(domain)) {
+      this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });
+    }
+    return this.buckets.get(domain)!;
+  }
+
+  private refill(domain: string, bucket: Bucket): void {
+    const rps = this.rateMap[domain] ?? this.defaultRps;
+    const now = Date.now();
+    const elapsed = (now - bucket.lastRefill) / 1000;
+    bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);
+    bucket.lastRefill = now;
+  }
+
+  /** Wait until a token is available for the given URL's domain. */
+  async throttle(url: string): Promise<void> {
+    const domain = extractDomain(url);
+    const bucket = this.getBucket(domain);
+    this.refill(domain, bucket);
+
+    if (bucket.tokens >= 1) {
+      bucket.tokens -= 1;
+      return;
+    }
+
+    // Calculate wait time until next token
+    const rps = this.rateMap[domain] ?? this.defaultRps;
+    const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1000);
+    await new Promise(r => setTimeout(r, waitMs));
+    bucket.tokens = 0;
+    bucket.lastRefill = Date.now();
+  }
+
+  /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
+  setRate(domain: string, rps: number): void {
+    this.rateMap[domain] = rps;
+    const bucket = this.getBucket(domain);
+    bucket.tokens = 0; // drain immediately on rate change
+  }
+
+  stats(): Record<string, { tokens: number; rps: number }> {
+    const result: Record<string, any> = {};
+    for (const [domain, bucket] of this.buckets) {
+      this.refill(domain, bucket);
+      result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };
+    }
+    return result;
+  }
+}

package/src/redirect-tracker.ts

@@ -0,0 +1,97 @@
+/**
+ * Redirect chain tracker — records the full redirect chain for crawled URLs.
+ *
+ * Why: redirect chains (A→B→C) slow down pages; circular redirects crash crawlers;
+ * redirect loops indicate config errors. ZeTa records chains for SEO/perf audit.
+ *
+ * Integration: hook into Playwright's page.on('response') to see each redirect hop.
+ */
+
+export interface RedirectHop {
+  from: string;
+  to: string;
+  statusCode: number;
+  durationMs: number;
+}
+
+export interface RedirectChain {
+  originalUrl: string;
+  finalUrl: string;
+  hops: RedirectHop[];
+  isCircular: boolean;
+  hopCount: number;
+  totalDurationMs: number;
+}
+
+import type { Page } from 'playwright';
+
+export async function trackRedirects(page: Page, url: string): Promise<RedirectChain> {
+  const hops: RedirectHop[] = [];
+  const seen = new Set<string>();
+  let lastResponseTime = Date.now();
+
+  const onResponse = (response: any) => {
+    const status = response.status();
+    if (status >= 300 && status < 400) {
+      const from = response.url();
+      const to = response.headers()['location'] ?? '';
+      const now = Date.now();
+      hops.push({ from, to, statusCode: status, durationMs: now - lastResponseTime });
+      lastResponseTime = now;
+    }
+  };
+
+  page.on('response', onResponse);
+
+  try {
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});
+  } finally {
+    page.off('response', onResponse);
+  }
+
+  const finalUrl = page.url();
+  const isCircular = hops.some(h => {
+    if (seen.has(h.to)) return true;
+    seen.add(h.from);
+    return false;
+  });
+
+  return {
+    originalUrl: url,
+    finalUrl,
+    hops,
+    isCircular,
+    hopCount: hops.length,
+    totalDurationMs: hops.reduce((sum, h) => sum + h.durationMs, 0),
+  };
+}
+
+export class RedirectRegistry {
+  private chains = new Map<string, RedirectChain>();
+
+  register(chain: RedirectChain): void {
+    this.chains.set(chain.originalUrl, chain);
+  }
+
+  get(url: string): RedirectChain | undefined {
+    return this.chains.get(url);
+  }
+
+  getAll(): RedirectChain[] {
+    return [...this.chains.values()];
+  }
+
+  getProblematic(): RedirectChain[] {
+    return [...this.chains.values()].filter(c => c.isCircular || c.hopCount > 3);
+  }
+
+  stats(): { total: number; withRedirects: number; circular: number; deepChains: number } {
+    const all = [...this.chains.values()];
+    return {
+      total: all.length,
+      withRedirects: all.filter(c => c.hopCount > 0).length,
+      circular: all.filter(c => c.isCircular).length,
+      deepChains: all.filter(c => c.hopCount > 3).length,
+    };
+  }
+}

package/src/redis-frontier.ts

@@ -0,0 +1,143 @@
+/**
+ * Redis-backed distributed crawl frontier.
+ *
+ * Replaces the in-memory CrawlFrontier for horizontally-scaled crawler deployments.
+ * Multiple crawler workers on different containers can share the same URL queue for
+ * a given job — enabling 10x crawl throughput on large enterprise apps.
+ *
+ * Redis key namespace:  zeta:frontier:{jobId}
+ *   LIST  zeta:frontier:{jobId}:queue   — BFS queue (LPUSH/RPOP)
+ *   SET   zeta:frontier:{jobId}:visited — deduplication set
+ *   EXPIRE                               — 24h TTL on all keys
+ *
+ * Graceful fallback: if REDIS_URL is not set, RedisCrawlFrontier silently falls
+ * back to in-memory arrays so local development works without Redis.
+ */
+
+const FRONTIER_TTL = 86400; // 24h in seconds
+
+export class RedisCrawlFrontier {
+  private redis: any;
+  private queueKey: string;
+  private visitedKey: string;
+  private inMemoryQueue: string[] = [];
+  private inMemoryVisited = new Set<string>();
+  private useRedis: boolean;
+
+  constructor(private readonly jobId: string) {
+    this.queueKey = `zeta:frontier:${jobId}:queue`;
+    this.visitedKey = `zeta:frontier:${jobId}:visited`;
+    this.useRedis = !!process.env.REDIS_URL;
+  }
+
+  async init(): Promise<void> {
+    if (!this.useRedis) return;
+    try {
+      const Redis: any = await import('ioredis').then(m => m.default ?? m);
+      this.redis = new Redis(process.env.REDIS_URL!, {
+        maxRetriesPerRequest: 2,
+        enableReadyCheck: false,
+        lazyConnect: true,
+      });
+      await this.redis.connect().catch(() => {
+        this.useRedis = false;
+        this.redis = null;
+      });
+    } catch {
+      this.useRedis = false;
+      this.redis = null;
+    }
+  }
+
+  /** Add a URL to the frontier. No-op if already visited. Returns true if added. */
+  async push(url: string): Promise<boolean> {
+    if (this.useRedis && this.redis) {
+      try {
+        const isNew = await this.redis.sadd(this.visitedKey, url);
+        if (!isNew) return false;
+        await this.redis.lpush(this.queueKey, url);
+        await this.redis.expire(this.queueKey, FRONTIER_TTL);
+        await this.redis.expire(this.visitedKey, FRONTIER_TTL);
+        return true;
+      } catch {
+        // Fall through to in-memory on Redis error
+      }
+    }
+    if (this.inMemoryVisited.has(url)) return false;
+    this.inMemoryVisited.add(url);
+    this.inMemoryQueue.push(url);
+    return true;
+  }
+
+  /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */
+  async shift(): Promise<string | null> {
+    if (this.useRedis && this.redis) {
+      try {
+        const url = await this.redis.rpop(this.queueKey);
+        return url ?? null;
+      } catch {
+        // Fall through
+      }
+    }
+    return this.inMemoryQueue.shift() ?? null;
+  }
+
+  /** Check if a URL has already been visited/queued. */
+  async hasVisited(url: string): Promise<boolean> {
+    if (this.useRedis && this.redis) {
+      try {
+        return (await this.redis.sismember(this.visitedKey, url)) === 1;
+      } catch { /* fall through */ }
+    }
+    return this.inMemoryVisited.has(url);
+  }
+
+  /** Current queue length. */
+  async length(): Promise<number> {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.llen(this.queueKey);
+      } catch { /* fall through */ }
+    }
+    return this.inMemoryQueue.length;
+  }
+
+  /** Total visited count. */
+  async visitedCount(): Promise<number> {
+    if (this.useRedis && this.redis) {
+      try {
+        return await this.redis.scard(this.visitedKey);
+      } catch { /* fall through */ }
+    }
+    return this.inMemoryVisited.size;
+  }
+
+  /** Seed initial URLs (used at job start). */
+  async seed(urls: string[]): Promise<void> {
+    for (const url of urls) await this.push(url);
+  }
+
+  /** Clean up all Redis keys for this frontier. Call at job completion. */
+  async cleanup(): Promise<void> {
+    if (this.useRedis && this.redis) {
+      try {
+        await this.redis.del(this.queueKey, this.visitedKey);
+      } catch { /* ignore */ }
+      try { await this.redis.quit(); } catch { /* ignore */ }
+    }
+  }
+
+  /** Stats for monitoring. */
+  async stats(): Promise<{ backend: 'redis' | 'memory'; queued: number; visited: number }> {
+    return {
+      backend: this.useRedis && this.redis ? 'redis' : 'memory',
+      queued: await this.length(),
+      visited: await this.visitedCount(),
+    };
+  }
+}
+
+/** Check if Redis is configured for distributed frontier. */
+export function isRedisFrontierEnabled(): boolean {
+  return !!process.env.REDIS_URL;
+}

package/src/redis-queue.ts

@@ -0,0 +1,208 @@
+/**
+ * Redis Streams distributed job queue.
+ *
+ * Provides horizontal crawler scaling via Redis Streams consumer groups.
+ * Multiple crawler nodes can pull from the same stream — each job is
+ * delivered to exactly one consumer (at-least-once via ACK/PENDING).
+ *
+ * Used when REDIS_URL is set AND JOB_QUEUE_BACKEND=redis.
+ * Falls back to the Postgres-backed queue when Redis is unavailable.
+ *
+ * Stream key:   zeta:crawl:stream
+ * Group name:   zeta:crawl:workers
+ * Consumer id:  zeta:worker:<hostname>:<pid>
+ */
+
+import { createHash } from 'node:crypto';
+import { hostname } from 'node:os';
+
+export interface RedisQueueJob {
+  id: string;        // Redis stream entry ID (e.g. "1720001234567-0")
+  jobId: string;     // Postgres job row ID
+  tenantId: string;
+  projectId: string;
+  payload: Record<string, unknown>;
+  enqueuedAt: string;
+}
+
+export interface RedisQueueClient {
+  enqueue(jobId: string, tenantId: string, projectId: string, payload?: Record<string, unknown>): Promise<string>;
+  dequeue(timeoutMs?: number): Promise<RedisQueueJob | null>;
+  ack(streamEntryId: string): Promise<void>;
+  nack(streamEntryId: string): Promise<void>;
+  reclaimStale(maxIdleMs?: number): Promise<RedisQueueJob[]>;
+  stats(): Promise<{ pending: number; delivered: number; groupLag: number }>;
+  close(): Promise<void>;
+}
+
+const STREAM_KEY = 'zeta:crawl:stream';
+const GROUP_NAME = 'zeta:crawl:workers';
+const MAX_LEN = 10_000; // approx cap on stream length
+
+function consumerId(): string {
+  return `zeta:worker:${hostname()}:${process.pid}`;
+}
+
+let ioredis: any;
+async function getIoredis() {
+  if (!ioredis) {
+    ioredis = await import('ioredis').then(m => m.default ?? m);
+  }
+  return ioredis;
+}
+
+/**
+ * Create a Redis Streams job queue client.
+ * Throws if REDIS_URL is not set.
+ */
+export async function createRedisQueue(): Promise<RedisQueueClient> {
+  const redisUrl = process.env.REDIS_URL;
+  if (!redisUrl) throw new Error('REDIS_URL env var required for redis queue backend');
+
+  const Redis = await getIoredis();
+  const redis = new Redis(redisUrl, {
+    maxRetriesPerRequest: 3,
+    enableReadyCheck: true,
+    lazyConnect: false,
+  });
+
+  // Create stream + consumer group if they don't exist
+  await redis.xgroup('CREATE', STREAM_KEY, GROUP_NAME, '$', 'MKSTREAM').catch((e: any) => {
+    if (!e.message?.includes('BUSYGROUP')) throw e;
+  });
+
+  const consumer = consumerId();
+
+  return {
+    async enqueue(jobId, tenantId, projectId, payload = {}): Promise<string> {
+      const entryId = await redis.xadd(
+        STREAM_KEY, 'MAXLEN', '~', MAX_LEN, '*',
+        'jobId', jobId,
+        'tenantId', tenantId,
+        'projectId', projectId,
+        'payload', JSON.stringify(payload),
+        'enqueuedAt', new Date().toISOString(),
+      );
+      return entryId as string;
+    },
+
+    async dequeue(timeoutMs = 5_000): Promise<RedisQueueJob | null> {
+      // First drain pending (unacknowledged) messages for this consumer
+      const pending: any[][] = await redis.xreadgroup(
+        'GROUP', GROUP_NAME, consumer,
+        'COUNT', '1',
+        'STREAMS', STREAM_KEY, '0',
+      ) ?? [];
+
+      let entries = pending?.[0]?.[1] ?? [];
+
+      if (!entries.length) {
+        // Block-read for new messages
+        const fresh: any[][] = await redis.xreadgroup(
+          'GROUP', GROUP_NAME, consumer,
+          'COUNT', '1',
+          'BLOCK', timeoutMs,
+          'STREAMS', STREAM_KEY, '>',
+        ) ?? [];
+        entries = fresh?.[0]?.[1] ?? [];
+      }
+
+      if (!entries.length) return null;
+
+      const [entryId, fields] = entries[0] as [string, string[]];
+      const obj: Record<string, string> = {};
+      for (let i = 0; i < fields.length; i += 2) {
+        obj[fields[i]] = fields[i + 1];
+      }
+
+      return {
+        id: entryId,
+        jobId: obj.jobId,
+        tenantId: obj.tenantId,
+        projectId: obj.projectId,
+        payload: JSON.parse(obj.payload ?? '{}'),
+        enqueuedAt: obj.enqueuedAt,
+      };
+    },
+
+    async ack(streamEntryId: string): Promise<void> {
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+
+    async nack(streamEntryId: string): Promise<void> {
+      // Move back to '>'-readable by deleting and re-adding (simplest approach)
+      // In production, prefer a dead-letter stream instead.
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+
+    async reclaimStale(maxIdleMs = 60_000): Promise<RedisQueueJob[]> {
+      const result: any = await redis.xautoclaim(
+        STREAM_KEY, GROUP_NAME, consumer,
+        maxIdleMs, '0-0', 'COUNT', '10',
+      );
+      const entries: [string, string[]][] = Array.isArray(result) ? (result[1] ?? []) : [];
+      return entries.map(([entryId, fields]) => {
+        const obj: Record<string, string> = {};
+        for (let i = 0; i < fields.length; i += 2) obj[fields[i]] = fields[i + 1];
+        return {
+          id: entryId,
+          jobId: obj.jobId,
+          tenantId: obj.tenantId,
+          projectId: obj.projectId,
+          payload: JSON.parse(obj.payload ?? '{}'),
+          enqueuedAt: obj.enqueuedAt,
+        };
+      });
+    },
+
+    async stats(): Promise<{ pending: number; delivered: number; groupLag: number }> {
+      const info: any[] = await redis.xinfo('GROUPS', STREAM_KEY);
+      // xinfo groups returns array of arrays: [name, ..., pel-count, ..., lag, ...]
+      const group = info.find((g: any) => {
+        const arr = Array.isArray(g) ? g : [];
+        for (let i = 0; i < arr.length; i += 2) {
+          if (arr[i] === 'name' && arr[i + 1] === GROUP_NAME) return true;
+        }
+        return false;
+      });
+
+      if (!group) return { pending: 0, delivered: 0, groupLag: 0 };
+
+      const flat = Array.isArray(group) ? group : [];
+      const get = (key: string) => {
+        for (let i = 0; i < flat.length - 1; i += 2) {
+          if (flat[i] === key) return Number(flat[i + 1]) || 0;
+        }
+        return 0;
+      };
+
+      return {
+        pending: get('pel-count'),
+        delivered: get('delivered-count'),
+        groupLag: get('lag'),
+      };
+    },
+
+    async close(): Promise<void> {
+      await redis.quit().catch(() => {});
+    },
+  };
+}
+
+/**
+ * Check if Redis queue backend is configured.
+ */
+export function isRedisQueueEnabled(): boolean {
+  return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === 'redis');
+}
+
+/**
+ * Convenience: create queue only if enabled, else return null.
+ */
+export async function maybeCreateRedisQueue(): Promise<RedisQueueClient | null> {
+  if (!isRedisQueueEnabled()) return null;
+  return createRedisQueue().catch((e) => {
+    console.error('[redis-queue] Failed to connect:', e.message);
+    return null;
+  });
+}

package/src/response-cache.ts

@@ -0,0 +1,150 @@
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+
+export interface ResponseCacheEntry {
+  url: string;
+  etag?: string;
+  lastModified?: string;
+  cacheControl?: string;
+  body: string;
+  contentType: string;
+  statusCode: number;
+  cachedAt: number;
+  hitCount: number;
+}
+
+export class ResponseCache {
+  private readonly dir: string;
+  private totalHits = 0;
+  private totalRequests = 0;
+
+  constructor(cacheDir: string) {
+    this.dir = cacheDir;
+    if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true });
+  }
+
+  private key(url: string): string {
+    return createHash('sha256').update(url).digest('hex').slice(0, 16);
+  }
+
+  private path(url: string): string {
+    return join(this.dir, `${this.key(url)}.json`);
+  }
+
+  get(url: string): ResponseCacheEntry | null {
+    this.totalRequests++;
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return null;
+      const entry: ResponseCacheEntry = JSON.parse(readFileSync(p, 'utf-8'));
+      this.totalHits++;
+      return entry;
+    } catch {
+      return null;
+    }
+  }
+
+  private read(url: string): ResponseCacheEntry | null {
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return null;
+      return JSON.parse(readFileSync(p, 'utf-8')) as ResponseCacheEntry;
+    } catch {
+      return null;
+    }
+  }
+
+  isFresh(url: string): boolean {
+    const entry = this.read(url);
+    if (!entry?.cacheControl) return false;
+    const m = /max-age=(\d+)/i.exec(entry.cacheControl);
+    if (!m) return false;
+    const maxAgeMs = parseInt(m[1], 10) * 1000;
+    return entry.cachedAt + maxAgeMs > Date.now();
+  }
+
+  staleIfError(url: string): boolean {
+    const entry = this.read(url);
+    if (!entry?.cacheControl) return false;
+    const m = /stale-if-error=(\d+)/i.exec(entry.cacheControl);
+    if (!m) return false;
+    const staleIfErrorMs = parseInt(m[1], 10) * 1000;
+    return entry.cachedAt + staleIfErrorMs >= Date.now();
+  }
+
+  set(url: string, entry: Omit<ResponseCacheEntry, 'cachedAt' | 'hitCount'>): void {
+    try {
+      const full: ResponseCacheEntry = { ...entry, cachedAt: Date.now(), hitCount: 0 };
+      writeFileSync(this.path(url), JSON.stringify(full), 'utf-8');
+    } catch { /* non-fatal */ }
+  }
+
+  getValidationHeaders(url: string): { 'If-None-Match'?: string; 'If-Modified-Since'?: string } {
+    const entry = this.get(url);
+    if (!entry) return {};
+    const headers: { 'If-None-Match'?: string; 'If-Modified-Since'?: string } = {};
+    if (entry.etag) headers['If-None-Match'] = entry.etag;
+    if (entry.lastModified) headers['If-Modified-Since'] = entry.lastModified;
+    return headers;
+  }
+
+  markHit(url: string): void {
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return;
+      const entry: ResponseCacheEntry = JSON.parse(readFileSync(p, 'utf-8'));
+      entry.hitCount++;
+      writeFileSync(p, JSON.stringify(entry), 'utf-8');
+    } catch { /* non-fatal */ }
+  }
+
+  invalidate(url: string): void {
+    try {
+      const p = this.path(url);
+      if (existsSync(p)) unlinkSync(p);
+    } catch { /* non-fatal */ }
+  }
+
+  clear(): void {
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (f.endsWith('.json')) unlinkSync(join(this.dir, f));
+      }
+    } catch { /* non-fatal */ }
+  }
+
+  stats(): { entries: number; totalSizeBytes: number; hitRate: number; totalHits: number } {
+    let entries = 0;
+    let totalSizeBytes = 0;
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (!f.endsWith('.json')) continue;
+        entries++;
+        try { totalSizeBytes += statSync(join(this.dir, f)).size; } catch { /* skip */ }
+      }
+    } catch { /* non-fatal */ }
+    return {
+      entries,
+      totalSizeBytes,
+      hitRate: this.totalRequests > 0 ? this.totalHits / this.totalRequests : 0,
+      totalHits: this.totalHits,
+    };
+  }
+
+  prune(maxAgeMs: number): number {
+    let deleted = 0;
+    const cutoff = Date.now() - maxAgeMs;
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (!f.endsWith('.json')) continue;
+        try {
+          const p = join(this.dir, f);
+          const entry: ResponseCacheEntry = JSON.parse(readFileSync(p, 'utf-8'));
+          if (entry.cachedAt < cutoff) { unlinkSync(p); deleted++; }
+        } catch { /* skip */ }
+      }
+    } catch { /* non-fatal */ }
+    return deleted;
+  }
+}