@detiq/crawler 0.1.3

package/src/http-engine.ts

@@ -0,0 +1,433 @@
+/**
+ * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
+ *
+ * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
+ * No browser overhead. Falls back to browser engine on JS-heavy sites.
+ *
+ * Features:
+ *   - HTTP/2 native (Node 20 built-in fetch)
+ *   - ETag / Last-Modified response caching
+ *   - 429 / 503 exponential backoff with jitter
+ *   - BFS / DFS traversal
+ *   - URL regex filtering
+ *   - Robots.txt compliance
+ *   - Link discovery (href, action, src)
+ *   - Markdown extraction (Readability + Turndown)
+ *   - OG / schema.org / meta extraction
+ *   - Table → JSON extraction
+ *   - JSON Lines, CSV, raw HTML output formats
+ *   - Streaming progress via onProgress callback
+ */
+
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+import { parseHTML } from 'linkedom';
+import { RobotsCache } from './robots-txt.js';
+import { ResponseCache } from './response-cache.js';
+import { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';
+import { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';
+import { renderingPredictor } from './engine-router.js';
+
+// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback
+let chromeFetchDispatcher: any;
+async function getChromeFetchDispatcher() {
+  if (!chromeFetchDispatcher) {
+    try {
+      // impit provides native Chrome BoringSSL TLS fingerprint — true JA3/JA4 match
+      // @ts-ignore — impit is an optional dependency
+      const impitMod = await import('impit');
+      const ImpitClass = impitMod.Impit ?? impitMod.default;
+      if (ImpitClass) {
+        chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });
+        return chromeFetchDispatcher;
+      }
+    } catch { /* impit not installed — fall through to undici */ }
+    const { Agent } = await import('undici');
+    chromeFetchDispatcher = new Agent({
+      connect: {
+        ciphers: CHROME_CIPHERS,
+        ecdhCurve: CHROME_CURVES,
+        minVersion: 'TLSv1.2' as any,
+        maxVersion: 'TLSv1.3' as any,
+      },
+      keepAliveTimeout: 10_000,
+      keepAliveMaxTimeout: 30_000,
+    });
+  }
+  return chromeFetchDispatcher;
+}
+
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+
+// Pages unlikely to have testable UI — skip from HTTP crawl queue
+const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
+
+export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
+
+export interface HttpScreen {
+  url: string;
+  title: string | null;
+  markdown: string;
+  html?: string;
+  metadata: {
+    description: string | null;
+    ogTitle: string | null;
+    ogDescription: string | null;
+    ogImage: string | null;
+    canonical: string | null;
+    schemaTypes: string[];
+    lang: string | null;
+  };
+  tables: Array<Record<string, string>[]>;
+  links: string[];
+  images: Array<{ src: string; alt: string }>;
+  statusCode: number;
+  fromCache: boolean;
+  requiresBrowser?: boolean;
+  durationMs: number;
+}
+
+export interface HttpCrawlOpts {
+  appUrl: string;
+  maxScreens?: number;            // default 100
+  urlFilter?: string;             // regex pattern — only crawl matching URLs
+  traversalStrategy?: 'BFS' | 'DFS'; // default BFS
+  outputFormats?: HttpOutputFormat[];
+  cacheDir?: string;              // default process.env.CRAWLER_CACHE_DIR || '/tmp/zeta-cache'
+  userAgent?: string;
+  headers?: Record<string, string>;
+  respectRobotsTxt?: boolean;     // default true
+  includeRawHtml?: boolean;
+  timeoutMs?: number;             // per-request timeout, default 15000
+  concurrency?: number;           // default 5 parallel fetches
+  onProgress?: (update: { url: string; screensFound: number; total: number; cached: boolean }) => void;
+}
+
+export interface HttpCrawlResult {
+  screens: HttpScreen[];
+  stats: {
+    total: number;
+    cached: number;
+    errors: number;
+    rateLimited: number;
+    durationMs: number;
+    cacheHitRate: number;
+  };
+  jsonLines?: string;
+  csv?: string;
+}
+
+function canonicalize(url: string): string {
+  try {
+    const u = new URL(url);
+    u.hash = '';
+    // Strip tracking params
+    for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {
+      u.searchParams.delete(p);
+    }
+    return u.href.toLowerCase().replace(/\/$/, '');
+  } catch {
+    return url.toLowerCase();
+  }
+}
+
+function extractLinks(html: string, baseUrl: string): string[] {
+  const links: string[] = [];
+  const seen = new Set<string>();
+  try {
+    const base = new URL(baseUrl);
+    const { document } = parseHTML(html);
+    const anchors = Array.from(document.querySelectorAll('a[href]'));
+    for (const a of anchors) {
+      const href = (a as any).getAttribute('href') ?? '';
+      if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;
+      try {
+        const resolved = new URL(href, base).href;
+        // Same origin only
+        if (new URL(resolved).origin !== base.origin) continue;
+        const c = canonicalize(resolved);
+        if (!seen.has(c)) { seen.add(c); links.push(resolved); }
+      } catch { /* skip */ }
+    }
+  } catch { /* non-fatal */ }
+  return links;
+}
+
+function extractMarkdown(html: string, url: string): { markdown: string; title: string | null } {
+  try {
+    const { document } = parseHTML(html);
+    // Remove script/style tags before Readability
+    for (const tag of Array.from(document.querySelectorAll('script, style, noscript'))) (tag as any).remove();
+    const reader = new Readability(document as any, { charThreshold: 50 });
+    const article = reader.parse();
+    if (article && article.textContent && article.textContent.split(/\s+/).length >= 30) {
+      const md = turndown.turndown(article.content ?? html);
+      return { markdown: md, title: article.title ?? null };
+    }
+  } catch { /* fall through */ }
+  // Full-page fallback
+  try {
+    const { document } = parseHTML(html);
+    for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer'))) (tag as any).remove();
+    const bodyHtml = (document as any).body?.innerHTML ?? html;
+    const md = turndown.turndown(bodyHtml);
+    const titleEl = (document as any).querySelector('title');
+    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };
+  } catch {
+    return { markdown: '', title: null };
+  }
+}
+
+interface FetchResult {
+  body: string;
+  statusCode: number;
+  contentType: string;
+  fromCache: boolean;
+  durationMs: number;
+  etag?: string;
+  lastModified?: string;
+}
+
+async function fetchWithBackoff(
+  url: string,
+  cache: ResponseCache,
+  opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },
+  retryCount = 0
+): Promise<FetchResult> {
+  const t0 = performance.now();
+
+  if (cache.isFresh(url)) {
+    const cached = cache.get(url)!;
+    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };
+  }
+
+  const validationHeaders = cache.getValidationHeaders(url);
+
+  const headers: Record<string, string> = {
+    'User-Agent': opts.userAgent,
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Accept-Encoding': 'gzip, deflate, br, zstd',
+    // Chrome-like security headers reduce bot-detection fingerprint
+    'sec-ch-ua': '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"Windows"',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'Upgrade-Insecure-Requests': '1',
+    ...validationHeaders,
+    ...(opts.extraHeaders ?? {}),
+  };
+
+  let response: Response;
+  try {
+    const dispatcher = await getChromeFetchDispatcher();
+    response = await (fetch as any)(url, {
+      headers,
+      signal: AbortSignal.timeout(opts.timeoutMs),
+      redirect: 'follow',
+      dispatcher,
+    });
+  } catch (e: any) {
+    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);
+  }
+
+  const durationMs = performance.now() - t0;
+
+  // 304 Not Modified — use cached body
+  if (response.status === 304) {
+    const cached = cache.get(url);
+    if (cached) {
+      cache.markHit(url);
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };
+    }
+  }
+
+  // 429 / 503 — exponential backoff with jitter
+  if (response.status === 429 || response.status === 503) {
+    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
+    const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;
+    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));
+    const jitter = Math.random() * 1000;
+    console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
+    await new Promise(r => setTimeout(r, baseDelay + jitter));
+    return fetchWithBackoff(url, cache, opts, retryCount + 1);
+  }
+
+  // 5xx — retry up to 3 times
+  if (response.status >= 500) {
+    if (retryCount < 3) {
+      await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));
+      return fetchWithBackoff(url, cache, opts, retryCount + 1);
+    }
+    if (cache.staleIfError(url)) {
+      const cached = cache.get(url)!;
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };
+    }
+  }
+
+  const contentType = response.headers.get('content-type') ?? '';
+  const body = await response.text();
+  const etag = response.headers.get('etag') ?? undefined;
+  const lastModified = response.headers.get('last-modified') ?? undefined;
+  const cacheControl = response.headers.get('cache-control') ?? undefined;
+
+  // Cache successful HTML responses
+  if (response.ok && contentType.includes('html')) {
+    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });
+  }
+
+  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };
+}
+
+export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
+  const startTime = performance.now();
+  const maxScreens = opts.maxScreens ?? 100;
+  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';
+  const timeoutMs = opts.timeoutMs ?? 15_000;
+  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zetaai.dpdns.org)';
+  const cache = new ResponseCache(cacheDir);
+  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
+  const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;
+
+  const visited = new Set<string>();
+  const queue: string[] = [opts.appUrl];
+  const queued = new Set<string>([canonicalize(opts.appUrl)]);
+  const screens: HttpScreen[] = [];
+  let cachedCount = 0;
+  let errorCount = 0;
+  let rateLimitedCount = 0;
+
+  const batchFetch = async (urls: string[]): Promise<void> => {
+    await Promise.all(urls.map(async (url) => {
+      const canonUrl = canonicalize(url);
+      if (visited.has(canonUrl)) return;
+      visited.add(canonUrl);
+
+      // Robots.txt check
+      if (robots && !(await robots.isAllowed(url).catch(() => true))) {
+        console.log(`[http-engine] robots.txt blocked: ${url}`);
+        return;
+      }
+
+      // Skip content/legal pages
+      try {
+        const { pathname } = new URL(url);
+        if (SKIP_PATH.test(pathname)) return;
+      } catch { /* skip */ }
+
+      // URL filter
+      if (urlFilterRe) {
+        try {
+          const { pathname, search } = new URL(url);
+          if (!urlFilterRe.test(pathname + search)) {
+            console.log(`[http-engine] filtered out: ${url}`);
+            return;
+          }
+        } catch { /* skip */ }
+      }
+
+      let fetchResult: FetchResult;
+      try {
+        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
+        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
+      } catch (err: any) {
+        console.error(`[http-engine] error fetching ${url}: ${err.message}`);
+        errorCount++;
+        return;
+      }
+
+      if (!fetchResult.contentType.includes('html')) return; // skip non-HTML
+      if (fetchResult.statusCode >= 400) return;
+      if (fetchResult.fromCache) cachedCount++;
+
+      const { markdown, title } = extractMarkdown(fetchResult.body, url);
+      const fullMeta = extractMetadata(fetchResult.body, url);
+      const links = extractLinks(fetchResult.body, url);
+      const tables = extractTables(fetchResult.body);
+      const images = extractImages(fetchResult.body, url);
+
+      const screen: HttpScreen = {
+        url,
+        title: title ?? fullMeta.title,
+        markdown,
+        html: opts.includeRawHtml ? fetchResult.body : undefined,
+        metadata: {
+          description: fullMeta.description,
+          ogTitle: fullMeta.ogTitle,
+          ogDescription: fullMeta.ogDescription,
+          ogImage: fullMeta.ogImage,
+          canonical: fullMeta.canonical,
+          schemaTypes: fullMeta.schemaTypes,
+          lang: fullMeta.lang,
+        },
+        tables,
+        links,
+        images: images.map(i => ({ src: i.src, alt: i.alt })),
+        statusCode: fetchResult.statusCode,
+        fromCache: fetchResult.fromCache,
+        requiresBrowser: renderingPredictor.getCached(url) === 'browser',
+        durationMs: fetchResult.durationMs,
+      };
+      screens.push(screen);
+
+      // Record rendering prediction: SPA signals → browser needed; content-rich → http engine works
+      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);
+
+      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });
+
+      // Enqueue discovered links
+      for (const link of links) {
+        const c = canonicalize(link);
+        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {
+          queued.add(c);
+          if (opts.traversalStrategy === 'DFS') queue.unshift(link);
+          else queue.push(link);
+        }
+      }
+    }));
+  };
+
+  const concurrency = opts.concurrency ?? 5;
+
+  while (queue.length > 0 && screens.length < maxScreens) {
+    const batch = queue.splice(0, concurrency);
+    await batchFetch(batch);
+  }
+
+  // Output formats
+  let jsonLines: string | undefined;
+  let csv: string | undefined;
+
+  const formats = opts.outputFormats ?? [];
+  if (formats.includes('jsonlines')) {
+    jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\n');
+  }
+  if (formats.includes('csv')) {
+    const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';
+    const rows = screens.map(s =>
+      [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `"${String(v).replace(/"/g, '""')}"`).join(',')
+    );
+    csv = [header, ...rows].join('\n');
+  }
+
+  const cacheStats = cache.stats();
+  return {
+    screens,
+    stats: {
+      total: screens.length,
+      cached: cachedCount,
+      errors: errorCount,
+      rateLimited: rateLimitedCount,
+      durationMs: performance.now() - startTime,
+      cacheHitRate: cacheStats.hitRate,
+    },
+    jsonLines,
+    csv,
+  };
+}

package/src/incremental-diff.ts

@@ -0,0 +1,121 @@
+/**
+ * Incremental crawl diff — compute stable content hashes and skip unchanged screens.
+ *
+ * Each crawled page gets a content fingerprint (murmurhash-free, pure JS).
+ * On the next crawl of the same project, pages with the same fingerprint are
+ * marked SKIPPED instead of re-processed, drastically reducing crawl time and
+ * LLM token spend for sites with mostly-static content.
+ *
+ * Design: "same content hash" means the meaningful text hasn't changed.
+ * Layout/CSS/JS changes that don't affect visible text are ignored intentionally.
+ */
+
+export interface ContentHash {
+  url: string;
+  hash: string;
+  textLength: number;
+  computedAt: string; // ISO timestamp
+}
+
+export interface DiffResult {
+  url: string;
+  changed: boolean;
+  previousHash?: string;
+  currentHash: string;
+  reason: 'new' | 'changed' | 'unchanged';
+}
+
+/**
+ * Compute a stable 64-bit-ish hash from text content.
+ * djb2 variant — fast, no deps, collision-resistant enough for content fingerprinting.
+ */
+export function hashContent(text: string): string {
+  let h1 = 5381;
+  let h2 = 52711;
+  for (let i = 0; i < text.length; i++) {
+    const c = text.charCodeAt(i);
+    h1 = (Math.imul(h1, 33) ^ c) >>> 0;
+    h2 = (Math.imul(h2, 31) ^ c) >>> 0;
+  }
+  return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');
+}
+
+/**
+ * Normalize text before hashing — strips whitespace-only lines, trims each line,
+ * collapses runs of blank lines. Makes hash stable across minor formatting changes.
+ */
+export function normalizeText(text: string): string {
+  return text
+    .split('\n')
+    .map(l => l.trim())
+    .filter(l => l.length > 0)
+    .join('\n');
+}
+
+export function computeContentHash(url: string, markdownText: string): ContentHash {
+  const normalized = normalizeText(markdownText);
+  return {
+    url,
+    hash: hashContent(normalized),
+    textLength: normalized.length,
+    computedAt: new Date().toISOString(),
+  };
+}
+
+/**
+ * Compare a page's current content against prior hashes.
+ * Returns { changed: false } for unchanged pages — caller should skip re-processing.
+ */
+export function diffContent(
+  url: string,
+  currentMarkdown: string,
+  priorHashes: Map<string, string>, // url → hash
+): DiffResult {
+  const current = computeContentHash(url, currentMarkdown);
+  const prior = priorHashes.get(url);
+
+  if (!prior) {
+    return { url, changed: true, currentHash: current.hash, reason: 'new' };
+  }
+  if (prior === current.hash) {
+    return { url, changed: false, previousHash: prior, currentHash: current.hash, reason: 'unchanged' };
+  }
+  return { url, changed: true, previousHash: prior, currentHash: current.hash, reason: 'changed' };
+}
+
+/**
+ * Build a hash map from prior screen records (from DB query result).
+ * screenRecords: Array<{ url: string; elementHash?: string | null }>
+ */
+export function buildPriorHashMap(screenRecords: Array<{ url: string; elementHash?: string | null }>): Map<string, string> {
+  const map = new Map<string, string>();
+  for (const s of screenRecords) {
+    if (s.url && s.elementHash) map.set(s.url, s.elementHash);
+  }
+  return map;
+}
+
+/**
+ * Filter a batch of discovered URLs against prior hashes.
+ * Returns { toProcess, toSkip } — caller enqueues toProcess, logs toSkip count.
+ *
+ * Note: filtering at URL level only works if the prior crawl already visited the URL.
+ * For new URLs, they're always in toProcess.
+ */
+export function partitionByChange(
+  urls: string[],
+  priorHashes: Map<string, string>,
+): { toProcess: string[]; toSkip: string[] } {
+  const toProcess: string[] = [];
+  const toSkip: string[] = [];
+  for (const url of urls) {
+    if (priorHashes.has(url)) {
+      // Can't know if changed without fetching — still needs fetch, but hash after fetch
+      // will determine skip. Include all — hash check happens post-fetch.
+      toProcess.push(url);
+    } else {
+      toProcess.push(url);
+    }
+  }
+  return { toProcess, toSkip };
+}

package/src/index.ts

@@ -0,0 +1,35 @@
+export * from './crawler.js';
+export * from './visual-diff-verdict.js';
+export * from './auth.js';
+export * from './screen-reader.js';
+export * from './job-queue.js';
+export { startSqsWorker } from './sqs-queue.js';
+export * from './session-capture.js';
+export * from './session-export.js';
+export * from './test-executor.js';
+export { startLiveView, getFrame, liveViewNavigate, completeLiveView, closeLiveView, sendInput } from './live-view.js';
+export { waitForEmailOTP, getInboxEmail } from './email-otp.js';
+export { getSsoToken, getSsoClientCredentialsToken } from './sso-token.js';
+export type { SsoConfig, SsoProvider } from './sso-token.js';
+export * from './engine-router.js';
+export * from './profiles/domain-profile.js';
+export * from './policies/crawl-safety.js';
+export * from './extractors/artifact-envelope.js';
+export * from './network-observer.js';
+export { createChromeTlsAgent, probeTlsFingerprint, chromeTlsAgent, CHROME_TLS_HEADERS } from './tls-fingerprint.js';
+export type { TlsFingerprintOptions } from './tls-fingerprint.js';
+export { prepareExtensionProfile, adBlockerProfile } from './browser-extension.js';
+export type { ExtensionSource, ExtensionProfile } from './browser-extension.js';
+export { createRedisQueue, maybeCreateRedisQueue, isRedisQueueEnabled } from './redis-queue.js';
+export type { RedisQueueJob, RedisQueueClient } from './redis-queue.js';
+export { discoverProjectUrls, discoverSitemapUrls } from './sitemap.js';
+export type { UrlCandidate } from './sitemap.js';
+export { httpCrawl } from './http-engine.js';
+export { extractMarkdown } from './markdown-extract.js';
+export type { ExtractedMarkdown } from './markdown-extract.js';
+export type { HttpScreen, HttpCrawlOpts, HttpCrawlResult, HttpOutputFormat } from './http-engine.js';
+export { TieredProxyManager } from './proxy.js';
+export type { ProxyTier, ProxyTierConfig } from './proxy.js';
+export { RedisCrawlFrontier, isRedisFrontierEnabled } from './redis-frontier.js';
+export { startSpan, endSpan, withSpan, increment, flushMetrics } from './telemetry.js';
+export type { Span, FinishedSpan } from './telemetry.js';