@taodigital/crawler 0.0.0-stage → 1.2.1

package/src/http-engine.ts

@@ -0,0 +1,453 @@
+/**
+ * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
+ *
+ * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
+ * No browser overhead. Falls back to browser engine on JS-heavy sites.
+ *
+ * Features:
+ *   - HTTP/2 native (Node 20 built-in fetch)
+ *   - ETag / Last-Modified response caching
+ *   - 429 / 503 exponential backoff with jitter
+ *   - BFS / DFS traversal
+ *   - URL regex filtering
+ *   - Robots.txt compliance
+ *   - Link discovery (href, action, src)
+ *   - Markdown extraction (Readability + Turndown)
+ *   - OG / schema.org / meta extraction
+ *   - Table → JSON extraction
+ *   - JSON Lines, CSV, raw HTML output formats
+ *   - Streaming progress via onProgress callback
+ */
+
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+import { parseHTML } from 'linkedom';
+import { RobotsCache } from './robots-txt.js';
+import { ResponseCache } from './response-cache.js';
+import { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';
+import { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';
+import { renderingPredictor } from './engine-router.js';
+
+// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback
+let chromeFetchDispatcher: any;
+async function getChromeFetchDispatcher() {
+  if (!chromeFetchDispatcher) {
+    try {
+      // impit provides native Chrome BoringSSL TLS fingerprint — true JA3/JA4 match
+      // @ts-ignore — impit is an optional dependency
+      const impitMod = await import('impit');
+      const ImpitClass = impitMod.Impit ?? impitMod.default;
+      if (ImpitClass) {
+        chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });
+        return chromeFetchDispatcher;
+      }
+    } catch { /* impit not installed — fall through to undici */ }
+    const { Agent } = await import('undici');
+    chromeFetchDispatcher = new Agent({
+      connect: {
+        ciphers: CHROME_CIPHERS,
+        ecdhCurve: CHROME_CURVES,
+        minVersion: 'TLSv1.2' as any,
+        maxVersion: 'TLSv1.3' as any,
+      },
+      keepAliveTimeout: 10_000,
+      keepAliveMaxTimeout: 30_000,
+    });
+  }
+  return chromeFetchDispatcher;
+}
+
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+
+// Pages unlikely to have testable UI — skip from HTTP crawl queue
+const _skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA
+  ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)
+  : [];
+const SKIP_PATH = _skipPathExtra.length > 0
+  ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join('|')})([-/]|$)`, 'i')
+  : /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+
+async function getHttpConfig() {
+  const { getSystemConfig } = await import('@taodigital/core');
+  const cfg = await getSystemConfig('crawl_tuning');
+  return {
+    retryLimit:   parseInt(String(cfg?.httpRetryLimit    ?? '5'),     10),
+    backoffCapMs: parseInt(String(cfg?.httpBackoffCapMs  ?? '60000'), 10),
+    retry5xx:     parseInt(String(cfg?.http5xxRetryLimit ?? '3'),     10),
+    timeoutMs:    parseInt(String(cfg?.httpTimeoutMs     ?? '15000'), 10),
+    concurrency:  parseInt(String(cfg?.httpConcurrency   ?? '5'),     10),
+  };
+}
+
+export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
+
+export interface HttpScreen {
+  url: string;
+  title: string | null;
+  markdown: string;
+  html?: string;
+  metadata: {
+    description: string | null;
+    ogTitle: string | null;
+    ogDescription: string | null;
+    ogImage: string | null;
+    canonical: string | null;
+    schemaTypes: string[];
+    lang: string | null;
+  };
+  tables: Array<Record<string, string>[]>;
+  links: string[];
+  images: Array<{ src: string; alt: string }>;
+  statusCode: number;
+  fromCache: boolean;
+  requiresBrowser?: boolean;
+  durationMs: number;
+}
+
+export interface HttpCrawlOpts {
+  appUrl: string;
+  maxScreens?: number;            // default 100
+  urlFilter?: string;             // regex pattern — only crawl matching URLs
+  traversalStrategy?: 'BFS' | 'DFS'; // default BFS
+  outputFormats?: HttpOutputFormat[];
+  cacheDir?: string;              // default '/tmp/zeta-cache'
+  userAgent?: string;
+  headers?: Record<string, string>;
+  respectRobotsTxt?: boolean;     // default true
+  includeRawHtml?: boolean;
+  timeoutMs?: number;             // per-request timeout, default 15000
+  concurrency?: number;           // default 5 parallel fetches
+  onProgress?: (update: { url: string; screensFound: number; total: number; cached: boolean }) => void;
+}
+
+export interface HttpCrawlResult {
+  screens: HttpScreen[];
+  stats: {
+    total: number;
+    cached: number;
+    errors: number;
+    rateLimited: number;
+    durationMs: number;
+    cacheHitRate: number;
+  };
+  jsonLines?: string;
+  csv?: string;
+}
+
+function canonicalize(url: string): string {
+  try {
+    const u = new URL(url);
+    u.hash = '';
+    // Strip tracking params
+    for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {
+      u.searchParams.delete(p);
+    }
+    return u.href.toLowerCase().replace(/\/$/, '');
+  } catch {
+    return url.toLowerCase();
+  }
+}
+
+function extractLinks(html: string, baseUrl: string): string[] {
+  const links: string[] = [];
+  const seen = new Set<string>();
+  try {
+    const base = new URL(baseUrl);
+    const { document } = parseHTML(html);
+    const anchors = Array.from(document.querySelectorAll('a[href]'));
+    for (const a of anchors) {
+      const href = (a as any).getAttribute('href') ?? '';
+      if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;
+      try {
+        const resolved = new URL(href, base).href;
+        // Same origin only
+        if (new URL(resolved).origin !== base.origin) continue;
+        const c = canonicalize(resolved);
+        if (!seen.has(c)) { seen.add(c); links.push(resolved); }
+      } catch { /* skip */ }
+    }
+  } catch { /* non-fatal */ }
+  return links;
+}
+
+function extractMarkdown(html: string, url: string): { markdown: string; title: string | null } {
+  try {
+    const { document } = parseHTML(html);
+    // Remove script/style tags before Readability
+    for (const tag of Array.from(document.querySelectorAll('script, style, noscript'))) (tag as any).remove();
+    const reader = new Readability(document as any, { charThreshold: 50 });
+    const article = reader.parse();
+    if (article && article.textContent && article.textContent.split(/\s+/).length >= 30) {
+      const md = turndown.turndown(article.content ?? html);
+      return { markdown: md, title: article.title ?? null };
+    }
+  } catch { /* fall through */ }
+  // Full-page fallback
+  try {
+    const { document } = parseHTML(html);
+    for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer'))) (tag as any).remove();
+    const bodyHtml = (document as any).body?.innerHTML ?? html;
+    const md = turndown.turndown(bodyHtml);
+    const titleEl = (document as any).querySelector('title');
+    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };
+  } catch {
+    return { markdown: '', title: null };
+  }
+}
+
+interface FetchResult {
+  body: string;
+  statusCode: number;
+  contentType: string;
+  fromCache: boolean;
+  durationMs: number;
+  etag?: string;
+  lastModified?: string;
+}
+
+async function fetchWithBackoff(
+  url: string,
+  cache: ResponseCache,
+  opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },
+  retryCount = 0,
+  httpCfg?: Awaited<ReturnType<typeof getHttpConfig>>
+): Promise<FetchResult> {
+  if (!httpCfg) httpCfg = await getHttpConfig();
+  const t0 = performance.now();
+
+  if (cache.isFresh(url)) {
+    const cached = cache.get(url)!;
+    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };
+  }
+
+  const validationHeaders = cache.getValidationHeaders(url);
+
+  const headers: Record<string, string> = {
+    'User-Agent': opts.userAgent,
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Accept-Encoding': 'gzip, deflate, br, zstd',
+    // Chrome-like security headers reduce bot-detection fingerprint
+    'sec-ch-ua': `"Google Chrome";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Chromium";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Not-A.Brand";v="99"`,
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"Windows"',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'Upgrade-Insecure-Requests': '1',
+    ...validationHeaders,
+    ...(opts.extraHeaders ?? {}),
+  };
+
+  let response: Response;
+  try {
+    const dispatcher = await getChromeFetchDispatcher();
+    response = await (fetch as any)(url, {
+      headers,
+      signal: AbortSignal.timeout(opts.timeoutMs),
+      redirect: 'follow',
+      dispatcher,
+    });
+  } catch (e: any) {
+    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);
+  }
+
+  const durationMs = performance.now() - t0;
+
+  // 304 Not Modified — use cached body
+  if (response.status === 304) {
+    const cached = cache.get(url);
+    if (cached) {
+      cache.markHit(url);
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };
+    }
+  }
+
+  // 429 / 503 — exponential backoff with jitter
+  if (response.status === 429 || response.status === 503) {
+    if (retryCount >= httpCfg.retryLimit) throw new Error(`[http-engine] rate-limited after ${httpCfg.retryLimit} retries: ${url}`);
+    const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;
+    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(httpCfg.backoffCapMs, 1_000 * Math.pow(2, retryCount));
+    const jitter = Math.random() * 1000;
+    console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/${httpCfg.retryLimit})`);
+    await new Promise(r => setTimeout(r, baseDelay + jitter));
+    return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
+  }
+
+  // 5xx — retry up to 3 times
+  if (response.status >= 500) {
+    if (retryCount < httpCfg.retry5xx) {
+      await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));
+      return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
+    }
+    if (cache.staleIfError(url)) {
+      const cached = cache.get(url)!;
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };
+    }
+  }
+
+  const contentType = response.headers.get('content-type') ?? '';
+  const body = await response.text();
+  const etag = response.headers.get('etag') ?? undefined;
+  const lastModified = response.headers.get('last-modified') ?? undefined;
+  const cacheControl = response.headers.get('cache-control') ?? undefined;
+
+  // Cache successful HTML responses
+  if (response.ok && contentType.includes('html')) {
+    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });
+  }
+
+  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };
+}
+
+export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
+  const startTime = performance.now();
+  const httpCfg = await getHttpConfig();
+  const maxScreens = opts.maxScreens ?? 100;
+  const cacheDir = opts.cacheDir ?? '/tmp/zeta-cache';
+  const timeoutMs = opts.timeoutMs ?? httpCfg.timeoutMs;
+  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';
+  const cache = new ResponseCache(cacheDir);
+  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
+  const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;
+
+  const visited = new Set<string>();
+  const queue: string[] = [opts.appUrl];
+  const queued = new Set<string>([canonicalize(opts.appUrl)]);
+  const screens: HttpScreen[] = [];
+  let cachedCount = 0;
+  let errorCount = 0;
+  let rateLimitedCount = 0;
+
+  const batchFetch = async (urls: string[]): Promise<void> => {
+    await Promise.all(urls.map(async (url) => {
+      const canonUrl = canonicalize(url);
+      if (visited.has(canonUrl)) return;
+      visited.add(canonUrl);
+
+      // Robots.txt check
+      if (robots && !(await robots.isAllowed(url).catch(() => true))) {
+        console.log(`[http-engine] robots.txt blocked: ${url}`);
+        return;
+      }
+
+      // Skip content/legal pages
+      try {
+        const { pathname } = new URL(url);
+        if (SKIP_PATH.test(pathname)) return;
+      } catch { /* skip */ }
+
+      // URL filter
+      if (urlFilterRe) {
+        try {
+          const { pathname, search } = new URL(url);
+          if (!urlFilterRe.test(pathname + search)) {
+            console.log(`[http-engine] filtered out: ${url}`);
+            return;
+          }
+        } catch { /* skip */ }
+      }
+
+      let fetchResult: FetchResult;
+      try {
+        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers }, 0, httpCfg);
+        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
+      } catch (err: any) {
+        console.error(`[http-engine] error fetching ${url}: ${err.message}`);
+        errorCount++;
+        return;
+      }
+
+      if (!fetchResult.contentType.includes('html')) return; // skip non-HTML
+      if (fetchResult.statusCode >= 400) return;
+      if (fetchResult.fromCache) cachedCount++;
+
+      const { markdown, title } = extractMarkdown(fetchResult.body, url);
+      const fullMeta = extractMetadata(fetchResult.body, url);
+      const links = extractLinks(fetchResult.body, url);
+      const tables = extractTables(fetchResult.body);
+      const images = extractImages(fetchResult.body, url);
+
+      const screen: HttpScreen = {
+        url,
+        title: title ?? fullMeta.title,
+        markdown,
+        html: opts.includeRawHtml ? fetchResult.body : undefined,
+        metadata: {
+          description: fullMeta.description,
+          ogTitle: fullMeta.ogTitle,
+          ogDescription: fullMeta.ogDescription,
+          ogImage: fullMeta.ogImage,
+          canonical: fullMeta.canonical,
+          schemaTypes: fullMeta.schemaTypes,
+          lang: fullMeta.lang,
+        },
+        tables,
+        links,
+        images: images.map(i => ({ src: i.src, alt: i.alt })),
+        statusCode: fetchResult.statusCode,
+        fromCache: fetchResult.fromCache,
+        requiresBrowser: renderingPredictor.getCached(url) === 'browser',
+        durationMs: fetchResult.durationMs,
+      };
+      screens.push(screen);
+
+      // Record rendering prediction: SPA signals → browser needed; content-rich → http engine works
+      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);
+
+      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });
+
+      // Enqueue discovered links
+      for (const link of links) {
+        const c = canonicalize(link);
+        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {
+          queued.add(c);
+          if (opts.traversalStrategy === 'DFS') queue.unshift(link);
+          else queue.push(link);
+        }
+      }
+    }));
+  };
+
+  const concurrency = opts.concurrency ?? httpCfg.concurrency;
+
+  while (queue.length > 0 && screens.length < maxScreens) {
+    const batch = queue.splice(0, concurrency);
+    await batchFetch(batch);
+  }
+
+  // Output formats
+  let jsonLines: string | undefined;
+  let csv: string | undefined;
+
+  const formats = opts.outputFormats ?? [];
+  if (formats.includes('jsonlines')) {
+    jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\n');
+  }
+  if (formats.includes('csv')) {
+    const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';
+    const rows = screens.map(s =>
+      [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `"${String(v).replace(/"/g, '""')}"`).join(',')
+    );
+    csv = [header, ...rows].join('\n');
+  }
+
+  const cacheStats = cache.stats();
+  return {
+    screens,
+    stats: {
+      total: screens.length,
+      cached: cachedCount,
+      errors: errorCount,
+      rateLimited: rateLimitedCount,
+      durationMs: performance.now() - startTime,
+      cacheHitRate: cacheStats.hitRate,
+    },
+    jsonLines,
+    csv,
+  };
+}