@detiq/crawler 1.1.9 → 1.1.11

package/dist/http-engine.js

@@ -1,353 +1,305 @@
-/**
- * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
- *
- * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
- * No browser overhead. Falls back to browser engine on JS-heavy sites.
- *
- * Features:
- *   - HTTP/2 native (Node 20 built-in fetch)
- *   - ETag / Last-Modified response caching
- *   - 429 / 503 exponential backoff with jitter
- *   - BFS / DFS traversal
- *   - URL regex filtering
- *   - Robots.txt compliance
- *   - Link discovery (href, action, src)
- *   - Markdown extraction (Readability + Turndown)
- *   - OG / schema.org / meta extraction
- *   - Table → JSON extraction
- *   - JSON Lines, CSV, raw HTML output formats
- *   - Streaming progress via onProgress callback
- */
-import { Readability } from '@mozilla/readability';
-import TurndownService from 'turndown';
-import { parseHTML } from 'linkedom';
-import { RobotsCache } from './robots-txt.js';
-import { ResponseCache } from './response-cache.js';
-import { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';
-import { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';
-import { renderingPredictor } from './engine-router.js';
-// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+import { parseHTML } from "linkedom";
+import { RobotsCache } from "./robots-txt.js";
+import { ResponseCache } from "./response-cache.js";
+import { extractMetadata, extractTables, extractImages } from "./extractors/advanced.js";
+import { CHROME_CIPHERS, CHROME_CURVES } from "./tls-fingerprint.js";
+import { renderingPredictor } from "./engine-router.js";
 let chromeFetchDispatcher;
 async function getChromeFetchDispatcher() {
-    if (!chromeFetchDispatcher) {
-        try {
-            // impit provides native Chrome BoringSSL TLS fingerprint — true JA3/JA4 match
-            // @ts-ignore — impit is an optional dependency
-            const impitMod = await import('impit');
-            const ImpitClass = impitMod.Impit ?? impitMod.default;
-            if (ImpitClass) {
-                chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });
-                return chromeFetchDispatcher;
-            }
-        }
-        catch { /* impit not installed — fall through to undici */ }
-        const { Agent } = await import('undici');
-        chromeFetchDispatcher = new Agent({
-            connect: {
-                ciphers: CHROME_CIPHERS,
-                ecdhCurve: CHROME_CURVES,
-                minVersion: 'TLSv1.2',
-                maxVersion: 'TLSv1.3',
-            },
-            keepAliveTimeout: 10_000,
-            keepAliveMaxTimeout: 30_000,
-        });
+  if (!chromeFetchDispatcher) {
+    try {
+      const impitMod = await import("impit");
+      const ImpitClass = impitMod.Impit ?? impitMod.default;
+      if (ImpitClass) {
+        chromeFetchDispatcher = new ImpitClass({ browser: "chrome" });
+        return chromeFetchDispatcher;
+      }
+    } catch {
     }
-    return chromeFetchDispatcher;
+    const { Agent } = await import("undici");
+    chromeFetchDispatcher = new Agent({
+      connect: {
+        ciphers: CHROME_CIPHERS,
+        ecdhCurve: CHROME_CURVES,
+        minVersion: "TLSv1.2",
+        maxVersion: "TLSv1.3"
+      },
+      keepAliveTimeout: 1e4,
+      keepAliveMaxTimeout: 3e4
+    });
+  }
+  return chromeFetchDispatcher;
 }
-const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
-// Pages unlikely to have testable UI — skip from HTTP crawl queue
+const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
 const SKIP_PATH = /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;
 function canonicalize(url) {
-    try {
-        const u = new URL(url);
-        u.hash = '';
-        // Strip tracking params
-        for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {
-            u.searchParams.delete(p);
-        }
-        return u.href.toLowerCase().replace(/\/$/, '');
-    }
-    catch {
-        return url.toLowerCase();
+  try {
+    const u = new URL(url);
+    u.hash = "";
+    for (const p of ["utm_source", "utm_medium", "utm_campaign", "utm_content", "utm_term", "fbclid", "gclid", "ref", "referrer"]) {
+      u.searchParams.delete(p);
     }
+    return u.href.toLowerCase().replace(/\/$/, "");
+  } catch {
+    return url.toLowerCase();
+  }
 }
 function extractLinks(html, baseUrl) {
-    const links = [];
-    const seen = new Set();
-    try {
-        const base = new URL(baseUrl);
-        const { document } = parseHTML(html);
-        const anchors = Array.from(document.querySelectorAll('a[href]'));
-        for (const a of anchors) {
-            const href = a.getAttribute('href') ?? '';
-            if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:'))
-                continue;
-            try {
-                const resolved = new URL(href, base).href;
-                // Same origin only
-                if (new URL(resolved).origin !== base.origin)
-                    continue;
-                const c = canonicalize(resolved);
-                if (!seen.has(c)) {
-                    seen.add(c);
-                    links.push(resolved);
-                }
-            }
-            catch { /* skip */ }
+  const links = [];
+  const seen = /* @__PURE__ */ new Set();
+  try {
+    const base = new URL(baseUrl);
+    const { document } = parseHTML(html);
+    const anchors = Array.from(document.querySelectorAll("a[href]"));
+    for (const a of anchors) {
+      const href = a.getAttribute("href") ?? "";
+      if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) continue;
+      try {
+        const resolved = new URL(href, base).href;
+        if (new URL(resolved).origin !== base.origin) continue;
+        const c = canonicalize(resolved);
+        if (!seen.has(c)) {
+          seen.add(c);
+          links.push(resolved);
         }
+      } catch {
+      }
     }
-    catch { /* non-fatal */ }
-    return links;
+  } catch {
+  }
+  return links;
 }
 function extractMarkdown(html, url) {
-    try {
-        const { document } = parseHTML(html);
-        // Remove script/style tags before Readability
-        for (const tag of Array.from(document.querySelectorAll('script, style, noscript')))
-            tag.remove();
-        const reader = new Readability(document, { charThreshold: 50 });
-        const article = reader.parse();
-        if (article && article.textContent && article.textContent.split(/\s+/).length >= 30) {
-            const md = turndown.turndown(article.content ?? html);
-            return { markdown: md, title: article.title ?? null };
-        }
-    }
-    catch { /* fall through */ }
-    // Full-page fallback
-    try {
-        const { document } = parseHTML(html);
-        for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer')))
-            tag.remove();
-        const bodyHtml = document.body?.innerHTML ?? html;
-        const md = turndown.turndown(bodyHtml);
-        const titleEl = document.querySelector('title');
-        return { markdown: md, title: titleEl?.textContent?.trim() ?? null };
-    }
-    catch {
-        return { markdown: '', title: null };
+  try {
+    const { document } = parseHTML(html);
+    for (const tag of Array.from(document.querySelectorAll("script, style, noscript"))) tag.remove();
+    const reader = new Readability(document, { charThreshold: 50 });
+    const article = reader.parse();
+    if (article && article.textContent && article.textContent.split(/\s+/).length >= 30) {
+      const md = turndown.turndown(article.content ?? html);
+      return { markdown: md, title: article.title ?? null };
     }
+  } catch {
+  }
+  try {
+    const { document } = parseHTML(html);
+    for (const tag of Array.from(document.querySelectorAll("script, style, noscript, nav, footer"))) tag.remove();
+    const bodyHtml = document.body?.innerHTML ?? html;
+    const md = turndown.turndown(bodyHtml);
+    const titleEl = document.querySelector("title");
+    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };
+  } catch {
+    return { markdown: "", title: null };
+  }
 }
 async function fetchWithBackoff(url, cache, opts, retryCount = 0) {
-    const t0 = performance.now();
-    if (cache.isFresh(url)) {
-        const cached = cache.get(url);
-        return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };
+  const t0 = performance.now();
+  if (cache.isFresh(url)) {
+    const cached = cache.get(url);
+    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };
+  }
+  const validationHeaders = cache.getValidationHeaders(url);
+  const headers = {
+    "User-Agent": opts.userAgent,
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br, zstd",
+    // Chrome-like security headers reduce bot-detection fingerprint
+    "sec-ch-ua": '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"Windows"',
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Upgrade-Insecure-Requests": "1",
+    ...validationHeaders,
+    ...opts.extraHeaders ?? {}
+  };
+  let response;
+  try {
+    const dispatcher = await getChromeFetchDispatcher();
+    response = await fetch(url, {
+      headers,
+      signal: AbortSignal.timeout(opts.timeoutMs),
+      redirect: "follow",
+      dispatcher
+    });
+  } catch (e) {
+    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);
+  }
+  const durationMs = performance.now() - t0;
+  if (response.status === 304) {
+    const cached = cache.get(url);
+    if (cached) {
+      cache.markHit(url);
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };
     }
-    const validationHeaders = cache.getValidationHeaders(url);
-    const headers = {
-        'User-Agent': opts.userAgent,
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
-        'Accept-Language': 'en-US,en;q=0.9',
-        'Accept-Encoding': 'gzip, deflate, br, zstd',
-        // Chrome-like security headers reduce bot-detection fingerprint
-        'sec-ch-ua': '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
-        'sec-ch-ua-mobile': '?0',
-        'sec-ch-ua-platform': '"Windows"',
-        'Sec-Fetch-Dest': 'document',
-        'Sec-Fetch-Mode': 'navigate',
-        'Sec-Fetch-Site': 'none',
-        'Sec-Fetch-User': '?1',
-        'Upgrade-Insecure-Requests': '1',
-        ...validationHeaders,
-        ...(opts.extraHeaders ?? {}),
-    };
-    let response;
-    try {
-        const dispatcher = await getChromeFetchDispatcher();
-        response = await fetch(url, {
-            headers,
-            signal: AbortSignal.timeout(opts.timeoutMs),
-            redirect: 'follow',
-            dispatcher,
-        });
+  }
+  if (response.status === 429 || response.status === 503) {
+    if (retryCount >= 5) throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
+    const retryAfter = parseInt(response.headers.get("retry-after") ?? "0") || 0;
+    const baseDelay = retryAfter > 0 ? retryAfter * 1e3 : Math.min(6e4, 1e3 * Math.pow(2, retryCount));
+    const jitter = Math.random() * 1e3;
+    console.log(`[http-engine] ${response.status} on ${url} \u2014 backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
+    await new Promise((r) => setTimeout(r, baseDelay + jitter));
+    return fetchWithBackoff(url, cache, opts, retryCount + 1);
+  }
+  if (response.status >= 500) {
+    if (retryCount < 3) {
+      await new Promise((r) => setTimeout(r, 2e3 * (retryCount + 1)));
+      return fetchWithBackoff(url, cache, opts, retryCount + 1);
     }
-    catch (e) {
-        throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);
+    if (cache.staleIfError(url)) {
+      const cached = cache.get(url);
+      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };
     }
-    const durationMs = performance.now() - t0;
-    // 304 Not Modified — use cached body
-    if (response.status === 304) {
-        const cached = cache.get(url);
-        if (cached) {
-            cache.markHit(url);
-            return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };
-        }
-    }
-    // 429 / 503 — exponential backoff with jitter
-    if (response.status === 429 || response.status === 503) {
-        if (retryCount >= 5)
-            throw new Error(`[http-engine] rate-limited after 5 retries: ${url}`);
-        const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;
-        const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(60_000, 1_000 * Math.pow(2, retryCount));
-        const jitter = Math.random() * 1000;
-        console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/5)`);
-        await new Promise(r => setTimeout(r, baseDelay + jitter));
-        return fetchWithBackoff(url, cache, opts, retryCount + 1);
-    }
-    // 5xx — retry up to 3 times
-    if (response.status >= 500) {
-        if (retryCount < 3) {
-            await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));
-            return fetchWithBackoff(url, cache, opts, retryCount + 1);
-        }
-        if (cache.staleIfError(url)) {
-            const cached = cache.get(url);
-            return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };
-        }
-    }
-    const contentType = response.headers.get('content-type') ?? '';
-    const body = await response.text();
-    const etag = response.headers.get('etag') ?? undefined;
-    const lastModified = response.headers.get('last-modified') ?? undefined;
-    const cacheControl = response.headers.get('cache-control') ?? undefined;
-    // Cache successful HTML responses
-    if (response.ok && contentType.includes('html')) {
-        cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });
-    }
-    return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };
+  }
+  const contentType = response.headers.get("content-type") ?? "";
+  const body = await response.text();
+  const etag = response.headers.get("etag") ?? void 0;
+  const lastModified = response.headers.get("last-modified") ?? void 0;
+  const cacheControl = response.headers.get("cache-control") ?? void 0;
+  if (response.ok && contentType.includes("html")) {
+    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });
+  }
+  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };
 }
-export async function httpCrawl(opts) {
-    const startTime = performance.now();
-    const maxScreens = opts.maxScreens ?? 100;
-    const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? '/tmp/zeta-cache';
-    const timeoutMs = opts.timeoutMs ?? 15_000;
-    const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? 'ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)';
-    const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
-    const cache = new ResponseCache(cacheDir);
-    const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
-    const urlFilterRe = opts.urlFilter ? (() => { try {
-        return new RegExp(opts.urlFilter, 'i');
-    }
-    catch {
-        return null;
-    } })() : null;
-    const visited = new Set();
-    const queue = [opts.appUrl];
-    const queued = new Set([canonicalize(opts.appUrl)]);
-    const screens = [];
-    let cachedCount = 0;
-    let errorCount = 0;
-    let rateLimitedCount = 0;
-    const batchFetch = async (urls) => {
-        await Promise.all(urls.map(async (url) => {
-            const canonUrl = canonicalize(url);
-            if (visited.has(canonUrl))
-                return;
-            visited.add(canonUrl);
-            // Robots.txt check
-            if (robots && !(await robots.isAllowed(url).catch(() => true))) {
-                console.log(`[http-engine] robots.txt blocked: ${url}`);
-                return;
-            }
-            // Skip content/legal pages
-            try {
-                const { pathname } = new URL(url);
-                if (SKIP_PATH.test(pathname))
-                    return;
-            }
-            catch { /* skip */ }
-            // URL filter
-            if (urlFilterRe) {
-                try {
-                    const { pathname, search } = new URL(url);
-                    if (!urlFilterRe.test(pathname + search)) {
-                        console.log(`[http-engine] filtered out: ${url}`);
-                        return;
-                    }
-                }
-                catch { /* skip */ }
-            }
-            let fetchResult;
-            try {
-                fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
-                if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503)
-                    rateLimitedCount++;
-            }
-            catch (err) {
-                console.error(`[http-engine] error fetching ${url}: ${err.message}`);
-                errorCount++;
-                return;
-            }
-            if (!fetchResult.contentType.includes('html'))
-                return; // skip non-HTML
-            if (fetchResult.statusCode >= 400)
-                return;
-            if (fetchResult.fromCache)
-                cachedCount++;
-            const { markdown, title } = extractMarkdown(fetchResult.body, url);
-            const fullMeta = extractMetadata(fetchResult.body, url);
-            const links = extractLinks(fetchResult.body, url);
-            const tables = extractTables(fetchResult.body);
-            const images = extractImages(fetchResult.body, url);
-            const screen = {
-                url,
-                title: title ?? fullMeta.title,
-                markdown,
-                html: opts.includeRawHtml ? fetchResult.body : undefined,
-                metadata: {
-                    description: fullMeta.description,
-                    ogTitle: fullMeta.ogTitle,
-                    ogDescription: fullMeta.ogDescription,
-                    ogImage: fullMeta.ogImage,
-                    canonical: fullMeta.canonical,
-                    schemaTypes: fullMeta.schemaTypes,
-                    lang: fullMeta.lang,
-                },
-                tables,
-                links,
-                images: images.map(i => ({ src: i.src, alt: i.alt })),
-                statusCode: fetchResult.statusCode,
-                fromCache: fetchResult.fromCache,
-                requiresBrowser: renderingPredictor.getCached(url) === 'browser',
-                durationMs: fetchResult.durationMs,
-            };
-            screens.push(screen);
-            // Record rendering prediction: SPA signals → browser needed; content-rich → http engine works
-            renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);
-            opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });
-            // Enqueue discovered links
-            for (const link of links) {
-                const c = canonicalize(link);
-                if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {
-                    queued.add(c);
-                    if (opts.traversalStrategy === 'DFS')
-                        queue.unshift(link);
-                    else
-                        queue.push(link);
-                }
-            }
-        }));
-    };
-    const concurrency = opts.concurrency ?? 5;
-    while (queue.length > 0 && screens.length < maxScreens) {
-        const batch = queue.splice(0, concurrency);
-        await batchFetch(batch);
-    }
-    // Output formats
-    let jsonLines;
-    let csv;
-    const formats = opts.outputFormats ?? [];
-    if (formats.includes('jsonlines')) {
-        jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\n');
-    }
-    if (formats.includes('csv')) {
-        const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';
-        const rows = screens.map(s => [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `"${String(v).replace(/"/g, '""')}"`).join(','));
-        csv = [header, ...rows].join('\n');
+async function httpCrawl(opts) {
+  const startTime = performance.now();
+  const maxScreens = opts.maxScreens ?? 100;
+  const cacheDir = opts.cacheDir ?? process.env.CRAWLER_CACHE_DIR ?? "/tmp/zeta-cache";
+  const timeoutMs = opts.timeoutMs ?? 15e3;
+  const CRAWLER_USER_AGENT = process.env.CRAWLER_USER_AGENT ?? "ZeTa-Crawler/1.0 (+https://zetaai.io/crawler)";
+  const userAgent = opts.userAgent ?? CRAWLER_USER_AGENT;
+  const cache = new ResponseCache(cacheDir);
+  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
+  const urlFilterRe = opts.urlFilter ? (() => {
+    try {
+      return new RegExp(opts.urlFilter, "i");
+    } catch {
+      return null;
     }
-    const cacheStats = cache.stats();
-    return {
-        screens,
-        stats: {
-            total: screens.length,
-            cached: cachedCount,
-            errors: errorCount,
-            rateLimited: rateLimitedCount,
-            durationMs: performance.now() - startTime,
-            cacheHitRate: cacheStats.hitRate,
+  })() : null;
+  const visited = /* @__PURE__ */ new Set();
+  const queue = [opts.appUrl];
+  const queued = /* @__PURE__ */ new Set([canonicalize(opts.appUrl)]);
+  const screens = [];
+  let cachedCount = 0;
+  let errorCount = 0;
+  let rateLimitedCount = 0;
+  const batchFetch = async (urls) => {
+    await Promise.all(urls.map(async (url) => {
+      const canonUrl = canonicalize(url);
+      if (visited.has(canonUrl)) return;
+      visited.add(canonUrl);
+      if (robots && !await robots.isAllowed(url).catch(() => true)) {
+        console.log(`[http-engine] robots.txt blocked: ${url}`);
+        return;
+      }
+      try {
+        const { pathname } = new URL(url);
+        if (SKIP_PATH.test(pathname)) return;
+      } catch {
+      }
+      if (urlFilterRe) {
+        try {
+          const { pathname, search } = new URL(url);
+          if (!urlFilterRe.test(pathname + search)) {
+            console.log(`[http-engine] filtered out: ${url}`);
+            return;
+          }
+        } catch {
+        }
+      }
+      let fetchResult;
+      try {
+        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers });
+        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
+      } catch (err) {
+        console.error(`[http-engine] error fetching ${url}: ${err.message}`);
+        errorCount++;
+        return;
+      }
+      if (!fetchResult.contentType.includes("html")) return;
+      if (fetchResult.statusCode >= 400) return;
+      if (fetchResult.fromCache) cachedCount++;
+      const { markdown, title } = extractMarkdown(fetchResult.body, url);
+      const fullMeta = extractMetadata(fetchResult.body, url);
+      const links = extractLinks(fetchResult.body, url);
+      const tables = extractTables(fetchResult.body);
+      const images = extractImages(fetchResult.body, url);
+      const screen = {
+        url,
+        title: title ?? fullMeta.title,
+        markdown,
+        html: opts.includeRawHtml ? fetchResult.body : void 0,
+        metadata: {
+          description: fullMeta.description,
+          ogTitle: fullMeta.ogTitle,
+          ogDescription: fullMeta.ogDescription,
+          ogImage: fullMeta.ogImage,
+          canonical: fullMeta.canonical,
+          schemaTypes: fullMeta.schemaTypes,
+          lang: fullMeta.lang
         },
-        jsonLines,
-        csv,
-    };
+        tables,
+        links,
+        images: images.map((i) => ({ src: i.src, alt: i.alt })),
+        statusCode: fetchResult.statusCode,
+        fromCache: fetchResult.fromCache,
+        requiresBrowser: renderingPredictor.getCached(url) === "browser",
+        durationMs: fetchResult.durationMs
+      };
+      screens.push(screen);
+      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);
+      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });
+      for (const link of links) {
+        const c = canonicalize(link);
+        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {
+          queued.add(c);
+          if (opts.traversalStrategy === "DFS") queue.unshift(link);
+          else queue.push(link);
+        }
+      }
+    }));
+  };
+  const concurrency = opts.concurrency ?? 5;
+  while (queue.length > 0 && screens.length < maxScreens) {
+    const batch = queue.splice(0, concurrency);
+    await batchFetch(batch);
+  }
+  let jsonLines;
+  let csv;
+  const formats = opts.outputFormats ?? [];
+  if (formats.includes("jsonlines")) {
+    jsonLines = screens.map((s) => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join("\n");
+  }
+  if (formats.includes("csv")) {
+    const header = "url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs";
+    const rows = screens.map(
+      (s) => [s.url, s.title ?? "", s.metadata.description ?? "", s.metadata.ogTitle ?? "", s.metadata.schemaTypes.join("|"), s.statusCode, s.fromCache, Math.round(s.durationMs)].map((v) => `"${String(v).replace(/"/g, '""')}"`).join(",")
+    );
+    csv = [header, ...rows].join("\n");
+  }
+  const cacheStats = cache.stats();
+  return {
+    screens,
+    stats: {
+      total: screens.length,
+      cached: cachedCount,
+      errors: errorCount,
+      rateLimited: rateLimitedCount,
+      durationMs: performance.now() - startTime,
+      cacheHitRate: cacheStats.hitRate
+    },
+    jsonLines,
+    csv
+  };
 }
+export {
+  httpCrawl
+};
+//# sourceMappingURL=http-engine.js.map