@detiq/crawler 0.1.3

package/src/redis-queue.ts

@@ -0,0 +1,208 @@
+/**
+ * Redis Streams distributed job queue.
+ *
+ * Provides horizontal crawler scaling via Redis Streams consumer groups.
+ * Multiple crawler nodes can pull from the same stream — each job is
+ * delivered to exactly one consumer (at-least-once via ACK/PENDING).
+ *
+ * Used when REDIS_URL is set AND JOB_QUEUE_BACKEND=redis.
+ * Falls back to the Postgres-backed queue when Redis is unavailable.
+ *
+ * Stream key:   zeta:crawl:stream
+ * Group name:   zeta:crawl:workers
+ * Consumer id:  zeta:worker:<hostname>:<pid>
+ */
+
+import { createHash } from 'node:crypto';
+import { hostname } from 'node:os';
+
+export interface RedisQueueJob {
+  id: string;        // Redis stream entry ID (e.g. "1720001234567-0")
+  jobId: string;     // Postgres job row ID
+  tenantId: string;
+  projectId: string;
+  payload: Record<string, unknown>;
+  enqueuedAt: string;
+}
+
+export interface RedisQueueClient {
+  enqueue(jobId: string, tenantId: string, projectId: string, payload?: Record<string, unknown>): Promise<string>;
+  dequeue(timeoutMs?: number): Promise<RedisQueueJob | null>;
+  ack(streamEntryId: string): Promise<void>;
+  nack(streamEntryId: string): Promise<void>;
+  reclaimStale(maxIdleMs?: number): Promise<RedisQueueJob[]>;
+  stats(): Promise<{ pending: number; delivered: number; groupLag: number }>;
+  close(): Promise<void>;
+}
+
+const STREAM_KEY = 'zeta:crawl:stream';
+const GROUP_NAME = 'zeta:crawl:workers';
+const MAX_LEN = 10_000; // approx cap on stream length
+
+function consumerId(): string {
+  return `zeta:worker:${hostname()}:${process.pid}`;
+}
+
+let ioredis: any;
+async function getIoredis() {
+  if (!ioredis) {
+    ioredis = await import('ioredis').then(m => m.default ?? m);
+  }
+  return ioredis;
+}
+
+/**
+ * Create a Redis Streams job queue client.
+ * Throws if REDIS_URL is not set.
+ */
+export async function createRedisQueue(): Promise<RedisQueueClient> {
+  const redisUrl = process.env.REDIS_URL;
+  if (!redisUrl) throw new Error('REDIS_URL env var required for redis queue backend');
+
+  const Redis = await getIoredis();
+  const redis = new Redis(redisUrl, {
+    maxRetriesPerRequest: 3,
+    enableReadyCheck: true,
+    lazyConnect: false,
+  });
+
+  // Create stream + consumer group if they don't exist
+  await redis.xgroup('CREATE', STREAM_KEY, GROUP_NAME, '$', 'MKSTREAM').catch((e: any) => {
+    if (!e.message?.includes('BUSYGROUP')) throw e;
+  });
+
+  const consumer = consumerId();
+
+  return {
+    async enqueue(jobId, tenantId, projectId, payload = {}): Promise<string> {
+      const entryId = await redis.xadd(
+        STREAM_KEY, 'MAXLEN', '~', MAX_LEN, '*',
+        'jobId', jobId,
+        'tenantId', tenantId,
+        'projectId', projectId,
+        'payload', JSON.stringify(payload),
+        'enqueuedAt', new Date().toISOString(),
+      );
+      return entryId as string;
+    },
+
+    async dequeue(timeoutMs = 5_000): Promise<RedisQueueJob | null> {
+      // First drain pending (unacknowledged) messages for this consumer
+      const pending: any[][] = await redis.xreadgroup(
+        'GROUP', GROUP_NAME, consumer,
+        'COUNT', '1',
+        'STREAMS', STREAM_KEY, '0',
+      ) ?? [];
+
+      let entries = pending?.[0]?.[1] ?? [];
+
+      if (!entries.length) {
+        // Block-read for new messages
+        const fresh: any[][] = await redis.xreadgroup(
+          'GROUP', GROUP_NAME, consumer,
+          'COUNT', '1',
+          'BLOCK', timeoutMs,
+          'STREAMS', STREAM_KEY, '>',
+        ) ?? [];
+        entries = fresh?.[0]?.[1] ?? [];
+      }
+
+      if (!entries.length) return null;
+
+      const [entryId, fields] = entries[0] as [string, string[]];
+      const obj: Record<string, string> = {};
+      for (let i = 0; i < fields.length; i += 2) {
+        obj[fields[i]] = fields[i + 1];
+      }
+
+      return {
+        id: entryId,
+        jobId: obj.jobId,
+        tenantId: obj.tenantId,
+        projectId: obj.projectId,
+        payload: JSON.parse(obj.payload ?? '{}'),
+        enqueuedAt: obj.enqueuedAt,
+      };
+    },
+
+    async ack(streamEntryId: string): Promise<void> {
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+
+    async nack(streamEntryId: string): Promise<void> {
+      // Move back to '>'-readable by deleting and re-adding (simplest approach)
+      // In production, prefer a dead-letter stream instead.
+      await redis.xack(STREAM_KEY, GROUP_NAME, streamEntryId);
+    },
+
+    async reclaimStale(maxIdleMs = 60_000): Promise<RedisQueueJob[]> {
+      const result: any = await redis.xautoclaim(
+        STREAM_KEY, GROUP_NAME, consumer,
+        maxIdleMs, '0-0', 'COUNT', '10',
+      );
+      const entries: [string, string[]][] = Array.isArray(result) ? (result[1] ?? []) : [];
+      return entries.map(([entryId, fields]) => {
+        const obj: Record<string, string> = {};
+        for (let i = 0; i < fields.length; i += 2) obj[fields[i]] = fields[i + 1];
+        return {
+          id: entryId,
+          jobId: obj.jobId,
+          tenantId: obj.tenantId,
+          projectId: obj.projectId,
+          payload: JSON.parse(obj.payload ?? '{}'),
+          enqueuedAt: obj.enqueuedAt,
+        };
+      });
+    },
+
+    async stats(): Promise<{ pending: number; delivered: number; groupLag: number }> {
+      const info: any[] = await redis.xinfo('GROUPS', STREAM_KEY);
+      // xinfo groups returns array of arrays: [name, ..., pel-count, ..., lag, ...]
+      const group = info.find((g: any) => {
+        const arr = Array.isArray(g) ? g : [];
+        for (let i = 0; i < arr.length; i += 2) {
+          if (arr[i] === 'name' && arr[i + 1] === GROUP_NAME) return true;
+        }
+        return false;
+      });
+
+      if (!group) return { pending: 0, delivered: 0, groupLag: 0 };
+
+      const flat = Array.isArray(group) ? group : [];
+      const get = (key: string) => {
+        for (let i = 0; i < flat.length - 1; i += 2) {
+          if (flat[i] === key) return Number(flat[i + 1]) || 0;
+        }
+        return 0;
+      };
+
+      return {
+        pending: get('pel-count'),
+        delivered: get('delivered-count'),
+        groupLag: get('lag'),
+      };
+    },
+
+    async close(): Promise<void> {
+      await redis.quit().catch(() => {});
+    },
+  };
+}
+
+/**
+ * Check if Redis queue backend is configured.
+ */
+export function isRedisQueueEnabled(): boolean {
+  return !!(process.env.REDIS_URL && process.env.JOB_QUEUE_BACKEND === 'redis');
+}
+
+/**
+ * Convenience: create queue only if enabled, else return null.
+ */
+export async function maybeCreateRedisQueue(): Promise<RedisQueueClient | null> {
+  if (!isRedisQueueEnabled()) return null;
+  return createRedisQueue().catch((e) => {
+    console.error('[redis-queue] Failed to connect:', e.message);
+    return null;
+  });
+}

package/src/response-cache.ts

@@ -0,0 +1,150 @@
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+
+export interface ResponseCacheEntry {
+  url: string;
+  etag?: string;
+  lastModified?: string;
+  cacheControl?: string;
+  body: string;
+  contentType: string;
+  statusCode: number;
+  cachedAt: number;
+  hitCount: number;
+}
+
+export class ResponseCache {
+  private readonly dir: string;
+  private totalHits = 0;
+  private totalRequests = 0;
+
+  constructor(cacheDir: string) {
+    this.dir = cacheDir;
+    if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true });
+  }
+
+  private key(url: string): string {
+    return createHash('sha256').update(url).digest('hex').slice(0, 16);
+  }
+
+  private path(url: string): string {
+    return join(this.dir, `${this.key(url)}.json`);
+  }
+
+  get(url: string): ResponseCacheEntry | null {
+    this.totalRequests++;
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return null;
+      const entry: ResponseCacheEntry = JSON.parse(readFileSync(p, 'utf-8'));
+      this.totalHits++;
+      return entry;
+    } catch {
+      return null;
+    }
+  }
+
+  private read(url: string): ResponseCacheEntry | null {
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return null;
+      return JSON.parse(readFileSync(p, 'utf-8')) as ResponseCacheEntry;
+    } catch {
+      return null;
+    }
+  }
+
+  isFresh(url: string): boolean {
+    const entry = this.read(url);
+    if (!entry?.cacheControl) return false;
+    const m = /max-age=(\d+)/i.exec(entry.cacheControl);
+    if (!m) return false;
+    const maxAgeMs = parseInt(m[1], 10) * 1000;
+    return entry.cachedAt + maxAgeMs > Date.now();
+  }
+
+  staleIfError(url: string): boolean {
+    const entry = this.read(url);
+    if (!entry?.cacheControl) return false;
+    const m = /stale-if-error=(\d+)/i.exec(entry.cacheControl);
+    if (!m) return false;
+    const staleIfErrorMs = parseInt(m[1], 10) * 1000;
+    return entry.cachedAt + staleIfErrorMs >= Date.now();
+  }
+
+  set(url: string, entry: Omit<ResponseCacheEntry, 'cachedAt' | 'hitCount'>): void {
+    try {
+      const full: ResponseCacheEntry = { ...entry, cachedAt: Date.now(), hitCount: 0 };
+      writeFileSync(this.path(url), JSON.stringify(full), 'utf-8');
+    } catch { /* non-fatal */ }
+  }
+
+  getValidationHeaders(url: string): { 'If-None-Match'?: string; 'If-Modified-Since'?: string } {
+    const entry = this.get(url);
+    if (!entry) return {};
+    const headers: { 'If-None-Match'?: string; 'If-Modified-Since'?: string } = {};
+    if (entry.etag) headers['If-None-Match'] = entry.etag;
+    if (entry.lastModified) headers['If-Modified-Since'] = entry.lastModified;
+    return headers;
+  }
+
+  markHit(url: string): void {
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return;
+      const entry: ResponseCacheEntry = JSON.parse(readFileSync(p, 'utf-8'));
+      entry.hitCount++;
+      writeFileSync(p, JSON.stringify(entry), 'utf-8');
+    } catch { /* non-fatal */ }
+  }
+
+  invalidate(url: string): void {
+    try {
+      const p = this.path(url);
+      if (existsSync(p)) unlinkSync(p);
+    } catch { /* non-fatal */ }
+  }
+
+  clear(): void {
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (f.endsWith('.json')) unlinkSync(join(this.dir, f));
+      }
+    } catch { /* non-fatal */ }
+  }
+
+  stats(): { entries: number; totalSizeBytes: number; hitRate: number; totalHits: number } {
+    let entries = 0;
+    let totalSizeBytes = 0;
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (!f.endsWith('.json')) continue;
+        entries++;
+        try { totalSizeBytes += statSync(join(this.dir, f)).size; } catch { /* skip */ }
+      }
+    } catch { /* non-fatal */ }
+    return {
+      entries,
+      totalSizeBytes,
+      hitRate: this.totalRequests > 0 ? this.totalHits / this.totalRequests : 0,
+      totalHits: this.totalHits,
+    };
+  }
+
+  prune(maxAgeMs: number): number {
+    let deleted = 0;
+    const cutoff = Date.now() - maxAgeMs;
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (!f.endsWith('.json')) continue;
+        try {
+          const p = join(this.dir, f);
+          const entry: ResponseCacheEntry = JSON.parse(readFileSync(p, 'utf-8'));
+          if (entry.cachedAt < cutoff) { unlinkSync(p); deleted++; }
+        } catch { /* skip */ }
+      }
+    } catch { /* non-fatal */ }
+    return deleted;
+  }
+}

package/src/responsive-detector.ts

@@ -0,0 +1,92 @@
+/**
+ * Responsive design detector — captures page at multiple breakpoints
+ * and detects layout differences between desktop and mobile viewports.
+ *
+ * Breakpoints: mobile(375), tablet(768), desktop(1280), wide(1920)
+ * Detects: viewport-specific content visibility, menu collapse state,
+ * font size changes, hidden/shown elements.
+ */
+
+import type { Page } from 'playwright';
+
+export interface ViewportCapture {
+  width: number;
+  height: number;
+  label: 'mobile' | 'tablet' | 'desktop' | 'wide';
+  screenshotBuffer?: Buffer;
+  visibleTextLength: number;
+  hasHamburgerMenu: boolean;
+  hiddenElements: number; // display:none element count
+  scrollHeight: number;
+}
+
+export interface ResponsiveReport {
+  url: string;
+  captures: ViewportCapture[];
+  isResponsive: boolean; // true if layout changes between viewports
+  hasMobileMenu: boolean;
+  issues: string[];
+}
+
+const BREAKPOINTS: Array<{ width: number; height: number; label: ViewportCapture['label'] }> = [
+  { width: 375, height: 812, label: 'mobile' },
+  { width: 768, height: 1024, label: 'tablet' },
+  { width: 1280, height: 800, label: 'desktop' },
+];
+
+export async function detectResponsive(
+  page: Page,
+  url: string,
+  opts: { captureScreenshots?: boolean } = {},
+): Promise<ResponsiveReport> {
+  const captures: ViewportCapture[] = [];
+  const issues: string[] = [];
+
+  for (const bp of BREAKPOINTS) {
+    try {
+      await page.setViewportSize({ width: bp.width, height: bp.height });
+      await page.waitForTimeout(500); // allow CSS reflow
+
+      const data = await page.evaluate(() => ({
+        visibleTextLength: (document.body.innerText ?? '').length,
+        hasHamburgerMenu: !!(
+          document.querySelector('[class*="hamburger"],[class*="menu-toggle"],[class*="nav-toggle"],[aria-label*="menu"],[aria-label*="Menu"]')
+        ),
+        hiddenElements: Array.from(document.querySelectorAll('*')).filter(
+          (el: any) => window.getComputedStyle(el).display === 'none'
+        ).length,
+        scrollHeight: document.body.scrollHeight,
+      })).catch(() => ({ visibleTextLength: 0, hasHamburgerMenu: false, hiddenElements: 0, scrollHeight: 0 }));
+
+      let screenshotBuffer: Buffer | undefined;
+      if (opts.captureScreenshots) {
+        screenshotBuffer = await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined);
+      }
+
+      captures.push({ ...bp, ...data, screenshotBuffer });
+    } catch { /* viewport failed — skip */ }
+  }
+
+  // Restore to desktop
+  await page.setViewportSize({ width: 1280, height: 800 }).catch(() => {});
+
+  const mobile = captures.find(c => c.label === 'mobile');
+  const desktop = captures.find(c => c.label === 'desktop');
+
+  const isResponsive = !!(mobile && desktop && (
+    mobile.hiddenElements !== desktop.hiddenElements ||
+    mobile.scrollHeight !== desktop.scrollHeight ||
+    mobile.hasHamburgerMenu
+  ));
+
+  const hasMobileMenu = captures.some(c => c.label === 'mobile' && c.hasHamburgerMenu);
+
+  if (mobile && desktop && mobile.visibleTextLength > desktop.visibleTextLength * 1.3) {
+    issues.push('Mobile shows significantly more text than desktop — possible responsive issue');
+  }
+  if (!isResponsive && captures.length > 1) {
+    issues.push('Layout appears identical across breakpoints — may not be responsive');
+  }
+
+  return { url, captures, isResponsive, hasMobileMenu, issues };
+}

package/src/retry-policy.ts

@@ -0,0 +1,87 @@
+/**
+ * Retry policy — configurable exponential backoff with jitter.
+ * Covers: 5xx server errors, network timeouts, ECONNREFUSED, ECONNRESET.
+ * NOT retried: 4xx client errors (except 429 — handled separately), redirects.
+ */
+
+export interface RetryPolicyOpts {
+  maxRetries?: number;       // default 3
+  baseDelayMs?: number;      // default 1000
+  maxDelayMs?: number;       // default 30000
+  jitterMs?: number;         // default 500
+  retryOn?: number[];        // HTTP status codes to retry (default: [429, 500, 502, 503, 504])
+}
+
+export interface RetryResult<T> {
+  value: T;
+  attempts: number;
+  totalDelayMs: number;
+}
+
+const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
+
+function shouldRetry(error: any, retryOn: number[]): boolean {
+  // Network-level errors
+  if (error?.code && ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EPIPE'].includes(error.code)) return true;
+  // Playwright navigation timeout
+  if (error?.message && /timeout|net::ERR_/i.test(error.message)) return true;
+  // HTTP status errors
+  if (typeof error?.status === 'number') return retryOn.includes(error.status);
+  return false;
+}
+
+function delay(ms: number): Promise<void> {
+  return new Promise(r => setTimeout(r, ms));
+}
+
+export async function withRetry<T>(
+  fn: (attempt: number) => Promise<T>,
+  opts: RetryPolicyOpts = {},
+): Promise<RetryResult<T>> {
+  const {
+    maxRetries = 3,
+    baseDelayMs = 1_000,
+    maxDelayMs = 30_000,
+    jitterMs = 500,
+    retryOn = DEFAULT_RETRY_STATUSES,
+  } = opts;
+
+  let lastError: any;
+  let totalDelayMs = 0;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const value = await fn(attempt);
+      return { value, attempts: attempt + 1, totalDelayMs };
+    } catch (err) {
+      lastError = err;
+      if (attempt === maxRetries) break;
+      if (!shouldRetry(err, retryOn)) throw err; // non-retryable — fail fast
+
+      const backoff = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));
+      const jitter = Math.random() * jitterMs;
+      const waitMs = Math.round(backoff + jitter);
+      totalDelayMs += waitMs;
+      await delay(waitMs);
+    }
+  }
+  throw lastError;
+}
+
+/** Wraps fetch() with retry policy */
+export async function fetchWithRetry(
+  url: string,
+  init: RequestInit = {},
+  retryOpts: RetryPolicyOpts = {},
+): Promise<Response> {
+  const result = await withRetry(async () => {
+    const res = await fetch(url, init);
+    if (retryOpts.retryOn?.includes(res.status) ?? DEFAULT_RETRY_STATUSES.includes(res.status)) {
+      const err: any = new Error(`HTTP ${res.status}`);
+      err.status = res.status;
+      throw err;
+    }
+    return res;
+  }, retryOpts);
+  return result.value;
+}

package/src/robots-txt.ts

@@ -0,0 +1,96 @@
+/**
+ * Minimal robots.txt fetcher + parser. No external deps. Caches per origin.
+ * Implements the subset of robots.txt spec used in practice:
+ *   User-agent, Disallow, Allow — longest match wins (RFC 9309).
+ */
+
+interface RobotsRule {
+  userAgent: string;
+  disallow: string[];
+  allow: string[];
+}
+
+function parseRobots(text: string): RobotsRule[] {
+  const rules: RobotsRule[] = [];
+  let current: RobotsRule | null = null;
+
+  for (const rawLine of text.split('\n')) {
+    const line = rawLine.split('#')[0].trim();
+    if (!line) { current = null; continue; }
+    const colonIdx = line.indexOf(':');
+    if (colonIdx === -1) continue;
+    const key = line.slice(0, colonIdx).trim().toLowerCase();
+    const value = line.slice(colonIdx + 1).trim();
+
+    if (key === 'user-agent') {
+      // Start new block only when current block already has rules, otherwise
+      // accumulate multiple User-agent lines into one block.
+      if (!current || current.disallow.length > 0 || current.allow.length > 0) {
+        current = { userAgent: value.toLowerCase(), disallow: [], allow: [] };
+        rules.push(current);
+      } else {
+        current.userAgent = value.toLowerCase();
+      }
+    } else if (key === 'disallow' && current && value) {
+      current.disallow.push(value);
+    } else if (key === 'allow' && current && value) {
+      current.allow.push(value);
+    }
+  }
+
+  return rules;
+}
+
+function isAllowedByRules(rules: RobotsRule[], userAgent: string, pathname: string): boolean {
+  const ua = userAgent.toLowerCase();
+  const exactRules = rules.filter(r => r.userAgent === ua);
+  const wildcardRules = rules.filter(r => r.userAgent === '*');
+  const applicable = exactRules.length > 0 ? exactRules : wildcardRules;
+
+  if (applicable.length === 0) return true;
+
+  // Collect all matched allow/disallow directives; longest path wins.
+  const matched: Array<{ path: string; allow: boolean }> = [];
+  for (const rule of applicable) {
+    for (const p of rule.allow) {
+      if (pathname.startsWith(p)) matched.push({ path: p, allow: true });
+    }
+    for (const p of rule.disallow) {
+      if (pathname.startsWith(p)) matched.push({ path: p, allow: false });
+    }
+  }
+
+  if (matched.length === 0) return true;
+  matched.sort((a, b) => b.path.length - a.path.length);
+  return matched[0].allow;
+}
+
+export class RobotsCache {
+  private readonly cache = new Map<string, RobotsRule[]>();
+
+  async isAllowed(url: string, userAgent = 'ZeTa-Crawler'): Promise<boolean> {
+    let origin: string;
+    let pathname: string;
+    try {
+      const u = new URL(url);
+      origin = u.origin;
+      pathname = u.pathname || '/';
+    } catch {
+      return true;
+    }
+
+    if (!this.cache.has(origin)) {
+      try {
+        const res = await fetch(`${origin}/robots.txt`, {
+          headers: { 'User-Agent': userAgent },
+          signal: AbortSignal.timeout(5_000),
+        });
+        this.cache.set(origin, res.ok ? parseRobots(await res.text()) : []);
+      } catch {
+        this.cache.set(origin, []); // network error or timeout = no restrictions
+      }
+    }
+
+    return isAllowedByRules(this.cache.get(origin)!, userAgent, pathname);
+  }
+}