@detiq/crawler 1.1.8 → 1.1.9

package/dist/http-engine.js

@@ -1,305 +1,353 @@
-import { Readability } from "@mozilla/readability";
-import TurndownService from "turndown";
-import { parseHTML } from "linkedom";
-import { RobotsCache } from "./robots-txt.js";
-import { ResponseCache } from "./response-cache.js";
-import { extractMetadata, extractTables, extractImages } from "./extractors/advanced.js";
-import { CHROME_CIPHERS, CHROME_CURVES } from "./tls-fingerprint.js";
-import { renderingPredictor } from "./engine-router.js";
+/**
+ * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
+ *
+ * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
+ * No browser overhead. Falls back to browser engine on JS-heavy sites.
+ *
+ * Features:
+ *   - HTTP/2 native (Node 20 built-in fetch)
+ *   - ETag / Last-Modified response caching
+ *   - 429 / 503 exponential backoff with jitter
+ *   - BFS / DFS traversal
+ *   - URL regex filtering
+ *   - Robots.txt compliance
+ *   - Link discovery (href, action, src)
+ *   - Markdown extraction (Readability + Turndown)
+ *   - OG / schema.org / meta extraction
+ *   - Table → JSON extraction
+ *   - JSON Lines, CSV, raw HTML output formats
+ *   - Streaming progress via onProgress callback
+ */
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+import { parseHTML } from 'linkedom';
+import { RobotsCache } from './robots-txt.js';
+import { ResponseCache } from './response-cache.js';
+import { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';
+import { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';
+import { renderingPredictor } from './engine-router.js';
+// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback
 let chromeFetchDispatcher;
 async function getChromeFetchDispatcher() {
-  if (!chromeFetchDispatcher) {
-    try {
-      const impitMod = await import("impit");
-      const ImpitClass = impitMod.Impit ?? impitMod.default;
-      if (ImpitClass) {
-        chromeFetchDispatcher = new ImpitClass({ browser: "chrome" });
-        return chromeFetchDispatcher;
-      }
-    } catch {
+    if (!chromeFetchDispatcher) {
+        try {
+            // impit provides native Chrome BoringSSL TLS fingerprint — true JA3/JA4 match
+            // @ts-ignore — impit is an optional dependency
+            const impitMod = await import('impit');
+            const ImpitClass = impitMod.Impit ?? impitMod.default;
+            if (ImpitClass) {
+                chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });
+                return chromeFetchDispatcher;
+            }
+        }
+        catch { /* impit not installed — fall through to undici */ }
+        const { Agent } = await import('undici');
+        chromeFetchDispatcher = new Agent({
+            connect: {
+                ciphers: CHROME_CIPHERS,
+                ecdhCurve: CHROME_CURVES,
+                minVersion: 'TLSv1.2',
+                maxVersion: 'TLSv1.3',
+            },
+            keepAliveTimeout: 10_000,
+            keepAliveMaxTimeout: 30_000,
+        });
     }
-    const { Agent } = await import("undici");
-    chromeFetchDispatcher = new Agent({
-      connect: {
-        ciphers: CHROME_CIPHERS,
-        ecdhCurve: CHROME_CURVES,
-        minVersion: "TLSv1.2",
-        maxVersion: "TLSv1.3"
-      },
-      keepAliveTimeout: 1e4,
-      keepAliveMaxTimeout: 3e4
-    });
-  }
-  return chromeFetchDispatcher;
+    return chromeFetchDispatcher;
 }
-const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+// Pages unlikely to have testable UI — skip from HTTP crawl queue
 const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
 function canonicalize(url) {
-  try {
-    const u = new URL(url);
-    u.hash = "";
-    for (const p of ["utm_source", "utm_medium", "utm_campaign", "utm_content", "utm_term", "fbclid", "gclid", "ref", "referrer"]) {
-      u.searchParams.delete(p);
+    try {
+        const u = new URL(url);
+        u.hash = '';
+        // Strip tracking params
+        for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {
+            u.searchParams.delete(p);
+        }
+        return u.href.toLowerCase().replace(/\/$/, '');
+    }
+    catch {
+        return url.toLowerCase();
     }
-    return u.href.toLowerCase().replace(/\/$/, "");
-  } catch {
-    return url.toLowerCase();
-  }
 }
 function extractLinks(html, baseUrl) {
-  const links = [];
-  const seen = /* @__PURE__ */ new Set();
-  try {
-    const base = new URL(baseUrl);
-    const { document } = parseHTML(html);
-    const anchors = Array.from(document.querySelectorAll("a[href]"));
-    for (const a of anchors) {
-      const href = a.getAttribute("href") ?? "";
-      if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) continue;
-      try {
-        const resolved = new URL(href, base).href;
-        if (new URL(resolved).origin !== base.origin) continue;
-        const c = canonicalize(resolved);
-        if (!seen.has(c)) {
-          seen.add(c);
-          links.push(resolved);
+    const links = [];
+    const seen = new Set();
+    try {
+        const base = new URL(baseUrl);
+        const { document } = parseHTML(html);
+        const anchors = Array.from(document.querySelectorAll('a[href]'));
+        for (const a of anchors) {
+            const href = a.getAttribute('href') ?? '';
+            if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:'))
+                continue;
+            try {
+                const resolved = new URL(href, base).href;
+                // Same origin only
+                if (new URL(resolved).origin !== base.origin)
+                    continue;
+                const c = canonicalize(resolved);
+                if (!seen.has(c)) {
+                    seen.add(c);
+                    links.push(resolved);
+                }
+            }
+            catch { /* skip */ }
         }
-      } catch {
-      }
     }
-  } catch {
-  }
-  return links;
+    catch { /* non-fatal */ }
+    return links;
 }
 function extractMarkdown(html, url) {
-  try {
-    const { document } = parseHTML(html);
-    for (const tag of Array.from(document.querySelectorAll("script, style, noscript"))) tag.remove();
-    const reader = new Readability(document, { charThreshold: 50 });
-    const article = reader.parse();
-    if (article && article.textContent && article.textContent.split(/\s+/).length >= 30) {
-      const md = turndown.turndown(article.content ?? html);
-      return { markdown: md, title: article.title ?? null };
+    try {
+        const { document } = parseHTML(html);
+        // Remove script/style tags before Readability
+        for (const tag of Array.from(document.querySelectorAll('script, style, noscript')))
+            tag.remove();
+        const reader = new Readability(document, { charThreshold: 50 });
+        const article = reader.parse();
+        if (article && article.textContent && article.textContent.split(/\s+/).length >= 30) {
+            const md = turndown.turndown(article.content ?? html);
+            return { markdown: md, title: article.title ?? null };
+        }
+    }
+    catch { /* fall through */ }
+    // Full-page fallback
+    try {
+        const { document } = parseHTML(html);
+        for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer')))
+            tag.remove();
+        const bodyHtml = document.body?.innerHTML ?? html;
+        const md = turndown.turndown(bodyHtml);
+        const titleEl = document.querySelector('title');
+        return { markdown: md, title: titleEl?.textContent?.trim() ?? null };
+    }
+    catch {
+        return { markdown: '', title: null };
     }
-  } catch {
-  }
-  try {
-    const { document } = parseHTML(html);
-    for (const tag of Array.from(document.querySelectorAll("script, style, noscript, nav, footer"))) tag.remove();
-    const bodyHtml = document.body?.innerHTML ?? html;
-    const md = turndown.turndown(bodyHtml);
-    const titleEl = document.querySelector("title");
-    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };
-  } catch {
-    return { markdown: "", title: null };
-  }
 }
 async function fetchWithBackoff(url, cache, opts, retryCount = 0) {
-  const t0 = performance.now();
-  if (cache.isFresh(url)) {
-    const cached = cache.get(url);
-    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };
-  }
-  const validationHeaders = cache.getValidationHeaders(url);
-  const headers = {
-    "User-Agent": opts.userAgent,
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-    "Accept-Language": "en-US,en;q=0.9",
-    "Accept-Encoding": "gzip, deflate, br, zstd",
-    // Chrome-like security headers reduce bot-detection fingerprint
-    "sec-ch-ua": '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
-    "sec-ch-ua-mobile": "?0",
-    "sec-ch-ua-platform": '"Windows"',
-    "Sec-Fetch-Dest": "document",
-    "Sec-Fetch-Mode": "navigate",
-    "Sec-Fetch-Site": "none",
-    "Sec-Fetch-User": "?1",
-    "Upgrade-Insecure-Requests": "1",
-    ...validationHeaders,
-    ...opts.extraHeaders ?? {}
-  };
-  let response;
-  try {
-    const dispatcher = await getChromeFetchDispatcher();
-    response = await fetch(url, {
-      headers,
-      signal: AbortSignal.timeout(opts.timeoutMs),
-      redirect: "follow",
-      dispatcher
-    });
-  } catch (e) {
-    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);
-  }
-  const durationMs = performance.now() - t0;
-  if (response.status === 304) {
-    const cached = cache.get(url);
-    if (cached) {
-      cache.markHit(url);
-      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };
+    const t0 = performance.now();
+    if (cache.isFresh(url)) {
+        const cached = cache.get(url);
+        return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };
     }
-  }
-  if (response.status === 429 || response.status === 503) {
-    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
-    const retryAfter = parseInt(response.headers.get("retry-after") ?? "0") || 0;
-    const baseDelay = retryAfter > 0 ? retryAfter * 1e3 : Math.min(6e4, 1e3 * Math.pow(2, retryCount));
-    const jitter = Math.random() * 1e3;
-    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
-    await new Promise((r) => setTimeout(r, baseDelay + jitter));
-    return fetchWithBackoff(url, cache, opts, retryCount + 1);
-  }
-  if (response.status >= 500) {
-    if (retryCount < 3) {
-      await new Promise((r) => setTimeout(r, 2e3 * (retryCount + 1)));
-      return fetchWithBackoff(url, cache, opts, retryCount + 1);
+    const validationHeaders = cache.getValidationHeaders(url);
+    const headers = {
+        'User-Agent': opts.userAgent,
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Accept-Encoding': 'gzip, deflate, br, zstd',
+        // Chrome-like security headers reduce bot-detection fingerprint
+        'sec-ch-ua': '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"Windows"',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-User': '?1',
+        'Upgrade-Insecure-Requests': '1',
+        ...validationHeaders,
+        ...(opts.extraHeaders ?? {}),
+    };
+    let response;
+    try {
+        const dispatcher = await getChromeFetchDispatcher();
+        response = await fetch(url, {
+            headers,
+            signal: AbortSignal.timeout(opts.timeoutMs),
+            redirect: 'follow',
+            dispatcher,
+        });
     }
-    if (cache.staleIfError(url)) {
-      const cached = cache.get(url);
-      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };
+    catch (e) {
+        throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);
     }
-  }
-  const contentType = response.headers.get("content-type") ?? "";
-  const body = await response.text();
-  const etag = response.headers.get("etag") ?? void 0;
-  const lastModified = response.headers.get("last-modified") ?? void 0;
-  const cacheControl = response.headers.get("cache-control") ?? void 0;
-  if (response.ok && contentType.includes("html")) {
-    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });
-  }
-  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };
-}
-async function httpCrawl(opts) {
-  const startTime = performance.now();
-  const maxScreens = opts.maxScreens ?? 100;
-  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? "/tmp/zeta-cache";
-  const timeoutMs = opts.timeoutMs ?? 15e3;
-  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? "ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)";
-  const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
-  const cache = new ResponseCache(cacheDir);
-  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
-  const urlFilterRe = opts.urlFilter ? (() => {
-    try {
-      return new RegExp(opts.urlFilter, "i");
-    } catch {
-      return null;
+    const durationMs = performance.now() - t0;
+    // 304 Not Modified — use cached body
+    if (response.status === 304) {
+        const cached = cache.get(url);
+        if (cached) {
+            cache.markHit(url);
+            return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };
+        }
     }
-  })() : null;
-  const visited = /* @__PURE__ */ new Set();
-  const queue = [opts.appUrl];
-  const queued = /* @__PURE__ */ new Set([canonicalize(opts.appUrl)]);
-  const screens = [];
-  let cachedCount = 0;
-  let errorCount = 0;
-  let rateLimitedCount = 0;
-  const batchFetch = async (urls) => {
-    await Promise.all(urls.map(async (url) => {
-      const canonUrl = canonicalize(url);
-      if (visited.has(canonUrl)) return;
-      visited.add(canonUrl);
-      if (robots && !await robots.isAllowed(url).catch(() => true)) {
-        console.log(`[http-engine] robots.txt blocked: ${url}`);
-        return;
-      }
-      try {
-        const { pathname } = new URL(url);
-        if (SKIP_PATH.test(pathname)) return;
-      } catch {
-      }
-      if (urlFilterRe) {
-        try {
-          const { pathname, search } = new URL(url);
-          if (!urlFilterRe.test(pathname + search)) {
-            console.log(`[http-engine] filtered out: ${url}`);
-            return;
-          }
-        } catch {
+    // 429 / 503 — exponential backoff with jitter
+    if (response.status === 429 || response.status === 503) {
+        if (retryCount >= 5)
+            throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
+        const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;
+        const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));
+        const jitter = Math.random() * 1000;
+        console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
+        await new Promise(r => setTimeout(r, baseDelay + jitter));
+        return fetchWithBackoff(url, cache, opts, retryCount + 1);
+    }
+    // 5xx — retry up to 3 times
+    if (response.status >= 500) {
+        if (retryCount < 3) {
+            await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));
+            return fetchWithBackoff(url, cache, opts, retryCount + 1);
         }
-      }
-      let fetchResult;
-      try {
-        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
-        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
-      } catch (err) {
-        console.error(`[http-engine] error fetching ${url}: ${err.message}`);
-        errorCount++;
-        return;
-      }
-      if (!fetchResult.contentType.includes("html")) return;
-      if (fetchResult.statusCode >= 400) return;
-      if (fetchResult.fromCache) cachedCount++;
-      const { markdown, title } = extractMarkdown(fetchResult.body, url);
-      const fullMeta = extractMetadata(fetchResult.body, url);
-      const links = extractLinks(fetchResult.body, url);
-      const tables = extractTables(fetchResult.body);
-      const images = extractImages(fetchResult.body, url);
-      const screen = {
-        url,
-        title: title ?? fullMeta.title,
-        markdown,
-        html: opts.includeRawHtml ? fetchResult.body : void 0,
-        metadata: {
-          description: fullMeta.description,
-          ogTitle: fullMeta.ogTitle,
-          ogDescription: fullMeta.ogDescription,
-          ogImage: fullMeta.ogImage,
-          canonical: fullMeta.canonical,
-          schemaTypes: fullMeta.schemaTypes,
-          lang: fullMeta.lang
-        },
-        tables,
-        links,
-        images: images.map((i) => ({ src: i.src, alt: i.alt })),
-        statusCode: fetchResult.statusCode,
-        fromCache: fetchResult.fromCache,
-        requiresBrowser: renderingPredictor.getCached(url) === "browser",
-        durationMs: fetchResult.durationMs
-      };
-      screens.push(screen);
-      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);
-      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });
-      for (const link of links) {
-        const c = canonicalize(link);
-        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {
-          queued.add(c);
-          if (opts.traversalStrategy === "DFS") queue.unshift(link);
-          else queue.push(link);
+        if (cache.staleIfError(url)) {
+            const cached = cache.get(url);
+            return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };
         }
-      }
-    }));
-  };
-  const concurrency = opts.concurrency ?? 5;
-  while (queue.length > 0 && screens.length < maxScreens) {
-    const batch = queue.splice(0, concurrency);
-    await batchFetch(batch);
-  }
-  let jsonLines;
-  let csv;
-  const formats = opts.outputFormats ?? [];
-  if (formats.includes("jsonlines")) {
-    jsonLines = screens.map((s) => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join("\n");
-  }
-  if (formats.includes("csv")) {
-    const header = "url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs";
-    const rows = screens.map(
-      (s) => [s.url, s.title ?? "", s.metadata.description ?? "", s.metadata.ogTitle ?? "", s.metadata.schemaTypes.join("|"), s.statusCode, s.fromCache, Math.round(s.durationMs)].map((v) => `"${String(v).replace(/"/g, '""')}"`).join(",")
-    );
-    csv = [header, ...rows].join("\n");
-  }
-  const cacheStats = cache.stats();
-  return {
-    screens,
-    stats: {
-      total: screens.length,
-      cached: cachedCount,
-      errors: errorCount,
-      rateLimited: rateLimitedCount,
-      durationMs: performance.now() - startTime,
-      cacheHitRate: cacheStats.hitRate
-    },
-    jsonLines,
-    csv
-  };
+    }
+    const contentType = response.headers.get('content-type') ?? '';
+    const body = await response.text();
+    const etag = response.headers.get('etag') ?? undefined;
+    const lastModified = response.headers.get('last-modified') ?? undefined;
+    const cacheControl = response.headers.get('cache-control') ?? undefined;
+    // Cache successful HTML responses
+    if (response.ok && contentType.includes('html')) {
+        cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });
+    }
+    return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };
+}
+export async function httpCrawl(opts) {
+    const startTime = performance.now();
+    const maxScreens = opts.maxScreens ?? 100;
+    const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';
+    const timeoutMs = opts.timeoutMs ?? 15_000;
+    const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)';
+    const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
+    const cache = new ResponseCache(cacheDir);
+    const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
+    const urlFilterRe = opts.urlFilter ? (() => { try {
+        return new RegExp(opts.urlFilter, 'i');
+    }
+    catch {
+        return null;
+    } })() : null;
+    const visited = new Set();
+    const queue = [opts.appUrl];
+    const queued = new Set([canonicalize(opts.appUrl)]);
+    const screens = [];
+    let cachedCount = 0;
+    let errorCount = 0;
+    let rateLimitedCount = 0;
+    const batchFetch = async (urls) => {
+        await Promise.all(urls.map(async (url) => {
+            const canonUrl = canonicalize(url);
+            if (visited.has(canonUrl))
+                return;
+            visited.add(canonUrl);
+            // Robots.txt check
+            if (robots && !(await robots.isAllowed(url).catch(() => true))) {
+                console.log(`[http-engine] robots.txt blocked: ${url}`);
+                return;
+            }
+            // Skip content/legal pages
+            try {
+                const { pathname } = new URL(url);
+                if (SKIP_PATH.test(pathname))
+                    return;
+            }
+            catch { /* skip */ }
+            // URL filter
+            if (urlFilterRe) {
+                try {
+                    const { pathname, search } = new URL(url);
+                    if (!urlFilterRe.test(pathname + search)) {
+                        console.log(`[http-engine] filtered out: ${url}`);
+                        return;
+                    }
+                }
+                catch { /* skip */ }
+            }
+            let fetchResult;
+            try {
+                fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
+                if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503)
+                    rateLimitedCount++;
+            }
+            catch (err) {
+                console.error(`[http-engine] error fetching ${url}: ${err.message}`);
+                errorCount++;
+                return;
+            }
+            if (!fetchResult.contentType.includes('html'))
+                return; // skip non-HTML
+            if (fetchResult.statusCode >= 400)
+                return;
+            if (fetchResult.fromCache)
+                cachedCount++;
+            const { markdown, title } = extractMarkdown(fetchResult.body, url);
+            const fullMeta = extractMetadata(fetchResult.body, url);
+            const links = extractLinks(fetchResult.body, url);
+            const tables = extractTables(fetchResult.body);
+            const images = extractImages(fetchResult.body, url);
+            const screen = {
+                url,
+                title: title ?? fullMeta.title,
+                markdown,
+                html: opts.includeRawHtml ? fetchResult.body : undefined,
+                metadata: {
+                    description: fullMeta.description,
+                    ogTitle: fullMeta.ogTitle,
+                    ogDescription: fullMeta.ogDescription,
+                    ogImage: fullMeta.ogImage,
+                    canonical: fullMeta.canonical,
+                    schemaTypes: fullMeta.schemaTypes,
+                    lang: fullMeta.lang,
+                },
+                tables,
+                links,
+                images: images.map(i => ({ src: i.src, alt: i.alt })),
+                statusCode: fetchResult.statusCode,
+                fromCache: fetchResult.fromCache,
+                requiresBrowser: renderingPredictor.getCached(url) === 'browser',
+                durationMs: fetchResult.durationMs,
+            };
+            screens.push(screen);
+            // Record rendering prediction: SPA signals → browser needed; content-rich → http engine works
+            renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);
+            opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });
+            // Enqueue discovered links
+            for (const link of links) {
+                const c = canonicalize(link);
+                if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {
+                    queued.add(c);
+                    if (opts.traversalStrategy === 'DFS')
+                        queue.unshift(link);
+                    else
+                        queue.push(link);
+                }
+            }
+        }));
+    };
+    const concurrency = opts.concurrency ?? 5;
+    while (queue.length > 0 && screens.length < maxScreens) {
+        const batch = queue.splice(0, concurrency);
+        await batchFetch(batch);
+    }
+    // Output formats
+    let jsonLines;
+    let csv;
+    const formats = opts.outputFormats ?? [];
+    if (formats.includes('jsonlines')) {
+        jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\n');
+    }
+    if (formats.includes('csv')) {
+        const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';
+        const rows = screens.map(s => [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `"${String(v).replace(/"/g, '""')}"`).join(','));
+        csv = [header, ...rows].join('\n');
+    }
+    const cacheStats = cache.stats();
+    return {
+        screens,
+        stats: {
+            total: screens.length,
+            cached: cachedCount,
+            errors: errorCount,
+            rateLimited: rateLimitedCount,
+            durationMs: performance.now() - startTime,
+            cacheHitRate: cacheStats.hitRate,
+        },
+        jsonLines,
+        csv,
+    };
 }
-export {
-  httpCrawl
-};
-//# sourceMappingURL=http-engine.js.map