@crawlith/core 0.1.0

package/dist/core/security/ipGuard.js

@@ -0,0 +1,84 @@
+import * as dns from 'dns';
+import * as net from 'net';
+import { promisify } from 'util';
+const resolve4 = promisify(dns.resolve4);
+const resolve6 = promisify(dns.resolve6);
+export class IPGuard {
+    /**
+     * Checks if an IP address is internal/private
+     */
+    static isInternal(ip) {
+        if (net.isIPv4(ip)) {
+            const parts = ip.split('.').map(Number);
+            // 127.0.0.0/8
+            if (parts[0] === 127)
+                return true;
+            // 10.0.0.0/8
+            if (parts[0] === 10)
+                return true;
+            // 192.168.0.0/16
+            if (parts[0] === 192 && parts[1] === 168)
+                return true;
+            // 172.16.0.0 – 172.31.255.255
+            if (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31)
+                return true;
+            // 169.254.0.0/16
+            if (parts[0] === 169 && parts[1] === 254)
+                return true;
+            // 0.0.0.0/8
+            if (parts[0] === 0)
+                return true;
+            return false;
+        }
+        if (net.isIPv6(ip)) {
+            // Normalize IPv6
+            const expanded = IPGuard.expandIPv6(ip);
+            // ::1
+            if (expanded === '0000:0000:0000:0000:0000:0000:0000:0001')
+                return true;
+            // fc00::/7 (Unique Local Address) -> fc or fd
+            const firstWord = parseInt(expanded.split(':')[0], 16);
+            if ((firstWord & 0xfe00) === 0xfc00)
+                return true;
+            // fe80::/10 (Link Local)
+            if ((firstWord & 0xffc0) === 0xfe80)
+                return true;
+            return false;
+        }
+        return true; // Unknown format, block it for safety
+    }
+    /**
+     * Resolves a hostname and validates all result IPs
+     */
+    static async validateHost(host) {
+        if (net.isIP(host)) {
+            return !IPGuard.isInternal(host);
+        }
+        try {
+            const res4 = await resolve4(host).catch(() => []);
+            const res6 = await resolve6(host).catch(() => []);
+            const ips = [...res4, ...res6];
+            if (ips.length === 0)
+                return true; // Let the fetcher handle DNS failures
+            return ips.every(ip => !IPGuard.isInternal(ip));
+        }
+        catch (_e) {
+            // If resolution fails drastically, we block for safety or let fetcher try
+            return false;
+        }
+    }
+    static expandIPv6(ip) {
+        if (ip === '::')
+            return '0000:0000:0000:0000:0000:0000:0000:0000';
+        let full = ip;
+        if (ip.includes('::')) {
+            const parts = ip.split('::');
+            const left = parts[0].split(':').filter(x => x !== '');
+            const right = parts[1].split(':').filter(x => x !== '');
+            const missing = 8 - (left.length + right.length);
+            const middle = Array(missing).fill('0000');
+            full = [...left, ...middle, ...right].join(':');
+        }
+        return full.split(':').map(part => part.padStart(4, '0')).join(':');
+    }
+}

package/dist/crawler/crawl.d.ts

@@ -0,0 +1,22 @@
+import { Graph } from '../graph/graph.js';
+export interface CrawlOptions {
+    limit: number;
+    depth: number;
+    concurrency?: number;
+    ignoreRobots?: boolean;
+    stripQuery?: boolean;
+    previousGraph?: Graph;
+    sitemap?: string;
+    debug?: boolean;
+    detectSoft404?: boolean;
+    detectTraps?: boolean;
+    rate?: number;
+    maxBytes?: number;
+    allowedDomains?: string[];
+    deniedDomains?: string[];
+    includeSubdomains?: boolean;
+    proxyUrl?: string;
+    maxRedirects?: number;
+    userAgent?: string;
+}
+export declare function crawl(startUrl: string, options: CrawlOptions): Promise<number>;

package/dist/crawler/crawl.js

@@ -0,0 +1,336 @@
+import { request } from 'undici';
+import pLimit from 'p-limit';
+import chalk from 'chalk';
+import robotsParser from 'robots-parser';
+import { Fetcher } from './fetcher.js';
+import { Parser } from './parser.js';
+import { Sitemap } from './sitemap.js';
+import { normalizeUrl } from './normalize.js';
+import { TrapDetector } from './trap.js';
+import { ScopeManager } from '../core/scope/scopeManager.js';
+import { getDb } from '../db/index.js';
+import { SiteRepository } from '../db/repositories/SiteRepository.js';
+import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
+import { PageRepository } from '../db/repositories/PageRepository.js';
+import { EdgeRepository } from '../db/repositories/EdgeRepository.js';
+import { MetricsRepository } from '../db/repositories/MetricsRepository.js';
+import { analyzeContent, calculateThinContentScore } from '../analysis/content.js';
+import { analyzeLinks } from '../analysis/links.js';
+export async function crawl(startUrl, options) {
+    const visited = new Set();
+    const concurrency = Math.min(options.concurrency || 2, 10);
+    const limitConcurrency = pLimit(concurrency);
+    const trapDetector = new TrapDetector();
+    const db = getDb();
+    const siteRepo = new SiteRepository(db);
+    const snapshotRepo = new SnapshotRepository(db);
+    const pageRepo = new PageRepository(db);
+    const edgeRepo = new EdgeRepository(db);
+    const metricsRepo = new MetricsRepository(db);
+    const rootUrl = normalizeUrl(startUrl, '', { stripQuery: options.stripQuery });
+    if (!rootUrl)
+        throw new Error('Invalid start URL');
+    const urlObj = new URL(rootUrl);
+    const domain = urlObj.hostname.replace('www.', '');
+    const site = siteRepo.firstOrCreateSite(domain);
+    const siteId = site.id;
+    const snapshotId = snapshotRepo.createSnapshot(siteId, options.previousGraph ? 'incremental' : 'full');
+    const rootOrigin = urlObj.origin;
+    // DB Helper
+    const savePageToDb = (url, depth, status, data = {}) => {
+        try {
+            const existing = pageRepo.getPage(siteId, url);
+            const isSameSnapshot = existing?.last_seen_snapshot_id === snapshotId;
+            return pageRepo.upsertAndGetId({
+                site_id: siteId,
+                normalized_url: url,
+                depth: isSameSnapshot ? existing.depth : depth,
+                http_status: status,
+                first_seen_snapshot_id: existing ? existing.first_seen_snapshot_id : snapshotId,
+                last_seen_snapshot_id: snapshotId,
+                canonical_url: data.canonical !== undefined ? data.canonical : existing?.canonical_url,
+                content_hash: data.contentHash !== undefined ? data.contentHash : existing?.content_hash,
+                simhash: data.simhash !== undefined ? data.simhash : existing?.simhash,
+                etag: data.etag !== undefined ? data.etag : existing?.etag,
+                last_modified: data.lastModified !== undefined ? data.lastModified : existing?.last_modified,
+                html: data.html !== undefined ? data.html : existing?.html,
+                soft404_score: data.soft404Score !== undefined ? data.soft404Score : existing?.soft404_score,
+                noindex: data.noindex !== undefined ? (data.noindex ? 1 : 0) : existing?.noindex,
+                nofollow: data.nofollow !== undefined ? (data.nofollow ? 1 : 0) : existing?.nofollow,
+                security_error: data.securityError !== undefined ? data.securityError : existing?.security_error,
+                retries: data.retries !== undefined ? data.retries : existing?.retries
+            });
+        }
+        catch (e) {
+            console.error(`Failed to save page ${url}:`, e);
+            return null;
+        }
+    };
+    const saveEdgeToDb = (sourceUrl, targetUrl, weight = 1.0, rel = 'internal') => {
+        try {
+            const sourceId = pageRepo.getIdByUrl(siteId, sourceUrl);
+            const targetId = pageRepo.getIdByUrl(siteId, targetUrl);
+            if (sourceId && targetId) {
+                edgeRepo.insertEdge(snapshotId, sourceId, targetId, weight, rel);
+            }
+        }
+        catch (e) {
+            console.error(`Failed to save edge ${sourceUrl} -> ${targetUrl}:`, e);
+        }
+    };
+    // Initialize Modules
+    const scopeManager = new ScopeManager({
+        allowedDomains: options.allowedDomains || [],
+        deniedDomains: options.deniedDomains || [],
+        includeSubdomains: options.includeSubdomains || false,
+        rootUrl: startUrl
+    });
+    const fetcher = new Fetcher({
+        rate: options.rate,
+        proxyUrl: options.proxyUrl,
+        scopeManager,
+        maxRedirects: options.maxRedirects,
+        userAgent: options.userAgent
+    });
+    const parser = new Parser();
+    const sitemapFetcher = new Sitemap();
+    // Handle robots.txt
+    let robots = null;
+    if (!options.ignoreRobots) {
+        try {
+            const robotsUrl = new URL('/robots.txt', rootOrigin).toString();
+            const res = await request(robotsUrl, {
+                maxRedirections: 3,
+                headers: { 'User-Agent': 'crawlith/1.0' },
+                headersTimeout: 5000,
+                bodyTimeout: 5000
+            });
+            if (res.statusCode >= 200 && res.statusCode < 300) {
+                const txt = await res.body.text();
+                robots = robotsParser(robotsUrl, txt);
+            }
+            else {
+                await res.body.dump();
+            }
+        }
+        catch {
+            console.warn('Failed to fetch robots.txt, proceeding...');
+        }
+    }
+    // Queue Setup
+    const queue = [];
+    const uniqueQueue = new Set();
+    const addToQueue = (u, d) => {
+        if (scopeManager.isUrlEligible(u) !== 'allowed')
+            return;
+        if (!uniqueQueue.has(u)) {
+            uniqueQueue.add(u);
+            queue.push({ url: u, depth: d });
+        }
+    };
+    // Seed from Sitemap
+    if (options.sitemap) {
+        try {
+            const sitemapUrl = options.sitemap === 'true' ? new URL('/sitemap.xml', rootOrigin).toString() : options.sitemap;
+            if (sitemapUrl.startsWith('http')) {
+                console.log(`Fetching sitemap: ${sitemapUrl}`);
+                const sitemapUrls = await sitemapFetcher.fetch(sitemapUrl);
+                for (const u of sitemapUrls) {
+                    const normalized = normalizeUrl(u, '', options);
+                    if (normalized)
+                        addToQueue(normalized, 0);
+                }
+            }
+        }
+        catch (e) {
+            console.warn('Sitemap fetch failed', e);
+        }
+    }
+    // Seed from startUrl
+    addToQueue(rootUrl, 0);
+    let pagesCrawled = 0;
+    let active = 0;
+    let reachedLimit = false;
+    const maxDepthInCrawl = Math.min(options.depth, 10);
+    const shouldEnqueue = (url, depth) => {
+        if (visited.has(url))
+            return false;
+        if (uniqueQueue.has(url))
+            return false;
+        if (depth > maxDepthInCrawl)
+            return false;
+        if (scopeManager.isUrlEligible(url) !== 'allowed')
+            return false;
+        if (options.detectTraps) {
+            const trap = trapDetector.checkTrap(url, depth);
+            if (trap.risk > 0.8)
+                return false;
+        }
+        return true;
+    };
+    return new Promise((resolve) => {
+        const checkDone = () => {
+            if (queue.length === 0 && active === 0) {
+                snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
+                    limit_reached: reachedLimit ? 1 : 0
+                });
+                resolve(snapshotId);
+                return true;
+            }
+            return false;
+        };
+        const next = () => {
+            if (checkDone())
+                return;
+            if (pagesCrawled >= options.limit) {
+                reachedLimit = true;
+                if (active === 0) {
+                    snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
+                        limit_reached: 1
+                    });
+                    resolve(snapshotId);
+                }
+                return;
+            }
+            while (queue.length > 0 && active < concurrency && pagesCrawled < options.limit) {
+                const item = queue.shift();
+                if (visited.has(item.url))
+                    continue;
+                if (robots && !robots.isAllowed(item.url, 'crawlith'))
+                    continue;
+                active++;
+                pagesCrawled++;
+                visited.add(item.url);
+                limitConcurrency(() => processPage(item)).finally(() => {
+                    active--;
+                    next();
+                });
+            }
+        };
+        const processPage = async (item) => {
+            const { url, depth } = item;
+            if (scopeManager.isUrlEligible(url) !== 'allowed') {
+                savePageToDb(url, depth, 0, { securityError: 'blocked_by_domain_filter' });
+                return;
+            }
+            const existingInDb = pageRepo.getPage(siteId, url);
+            savePageToDb(url, depth, 0);
+            try {
+                const res = await fetcher.fetch(url, {
+                    etag: existingInDb?.etag || undefined,
+                    lastModified: existingInDb?.last_modified || undefined,
+                    maxBytes: options.maxBytes,
+                    crawlDelay: robots ? robots.getCrawlDelay('crawlith') : undefined
+                });
+                if (options.debug) {
+                    console.log(`${chalk.gray(`[D:${depth}]`)} ${res.status} ${chalk.blue(url)}`);
+                }
+                if (res.status === 304) {
+                    savePageToDb(url, depth, 304);
+                    metricsRepo.insertMetrics({
+                        snapshot_id: snapshotId,
+                        page_id: existingInDb.id,
+                        authority_score: null,
+                        hub_score: null,
+                        pagerank: null,
+                        pagerank_score: null,
+                        link_role: null,
+                        crawl_status: 'cached',
+                        word_count: null,
+                        thin_content_score: null,
+                        external_link_ratio: null,
+                        orphan_score: null,
+                        duplicate_cluster_id: null,
+                        duplicate_type: null,
+                        is_cluster_primary: 0
+                    });
+                    return;
+                }
+                const chain = res.redirectChain;
+                for (const step of chain) {
+                    const source = normalizeUrl(step.url, '', options);
+                    const target = normalizeUrl(step.target, '', options);
+                    if (source && target) {
+                        savePageToDb(source, depth, step.status);
+                        savePageToDb(target, depth, 0);
+                        saveEdgeToDb(source, target);
+                    }
+                }
+                const finalUrl = normalizeUrl(res.finalUrl, '', options);
+                if (!finalUrl)
+                    return;
+                const isStringStatus = typeof res.status === 'string';
+                if (isStringStatus || (typeof res.status === 'number' && res.status >= 300)) {
+                    savePageToDb(finalUrl, depth, typeof res.status === 'number' ? res.status : 0, {
+                        securityError: isStringStatus ? res.status : undefined,
+                        retries: res.retries
+                    });
+                    return;
+                }
+                if (res.status === 200) {
+                    const contentTypeHeader = res.headers['content-type'];
+                    const contentType = Array.isArray(contentTypeHeader) ? contentTypeHeader[0] : (contentTypeHeader || '');
+                    if (!contentType || !contentType.toLowerCase().includes('text/html')) {
+                        savePageToDb(finalUrl, depth, res.status);
+                        return;
+                    }
+                    savePageToDb(finalUrl, depth, res.status);
+                    const parseResult = parser.parse(res.body, finalUrl, res.status);
+                    const pageId = savePageToDb(finalUrl, depth, res.status, {
+                        html: parseResult.html,
+                        canonical: parseResult.canonical || undefined,
+                        noindex: parseResult.noindex,
+                        nofollow: parseResult.nofollow,
+                        contentHash: parseResult.contentHash,
+                        simhash: parseResult.simhash,
+                        soft404Score: parseResult.soft404Score,
+                        etag: res.etag,
+                        lastModified: res.lastModified,
+                        retries: res.retries
+                    });
+                    if (pageId) {
+                        try {
+                            const contentAnalysis = analyzeContent(parseResult.html);
+                            const linkAnalysis = analyzeLinks(parseResult.html, finalUrl, rootOrigin);
+                            const thinScore = calculateThinContentScore(contentAnalysis, 0);
+                            metricsRepo.insertMetrics({
+                                snapshot_id: snapshotId,
+                                page_id: pageId,
+                                authority_score: null,
+                                hub_score: null,
+                                pagerank: null,
+                                pagerank_score: null,
+                                link_role: null,
+                                crawl_status: 'fetched',
+                                word_count: contentAnalysis.wordCount,
+                                thin_content_score: thinScore,
+                                external_link_ratio: linkAnalysis.externalRatio,
+                                orphan_score: null,
+                                duplicate_cluster_id: null,
+                                duplicate_type: null,
+                                is_cluster_primary: 0
+                            });
+                        }
+                        catch (e) {
+                            console.error(`Error calculating per-page metrics for ${finalUrl}:`, e);
+                        }
+                    }
+                    for (const linkItem of parseResult.links) {
+                        const normalizedLink = normalizeUrl(linkItem.url, '', options);
+                        if (normalizedLink && normalizedLink !== finalUrl) {
+                            savePageToDb(normalizedLink, depth + 1, 0);
+                            saveEdgeToDb(finalUrl, normalizedLink, 1.0, 'internal');
+                            if (shouldEnqueue(normalizedLink, depth + 1)) {
+                                addToQueue(normalizedLink, depth + 1);
+                            }
+                        }
+                    }
+                }
+            }
+            catch (e) {
+                console.error(`Error processing ${url}:`, e);
+            }
+        };
+        next();
+    });
+}

package/dist/crawler/extract.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Extracts all links from an HTML document.
+ * Returns absolute URLs.
+ */
+export declare function extractLinks(html: string, baseUrl: string): string[];

package/dist/crawler/extract.js

@@ -0,0 +1,33 @@
+import * as cheerio from 'cheerio';
+/**
+ * Extracts all links from an HTML document.
+ * Returns absolute URLs.
+ */
+export function extractLinks(html, baseUrl) {
+    try {
+        const $ = cheerio.load(html);
+        const links = new Set();
+        $('a').each((_, element) => {
+            const href = $(element).attr('href');
+            if (href) {
+                try {
+                    const absoluteUrl = new URL(href, baseUrl);
+                    // Only http(s) links
+                    if (absoluteUrl.protocol === 'http:' || absoluteUrl.protocol === 'https:') {
+                        // Remove hash fragments immediately as they are irrelevant for crawling
+                        absoluteUrl.hash = '';
+                        links.add(absoluteUrl.toString());
+                    }
+                }
+                catch (_e) {
+                    // Invalid URL, skip
+                }
+            }
+        });
+        return Array.from(links);
+    }
+    catch (e) {
+        console.error(`Error extracting links from ${baseUrl}:`, e);
+        return [];
+    }
+}

package/dist/crawler/fetcher.d.ts

@@ -0,0 +1,40 @@
+import { ScopeManager } from '../core/scope/scopeManager.js';
+export interface RedirectStep {
+    url: string;
+    status: number;
+    target: string;
+}
+export interface FetchResult {
+    status: number | 'blocked_internal_ip' | 'blocked_by_domain_filter' | 'blocked_subdomain' | 'oversized' | 'failed_after_retries' | 'network_error' | 'redirect_limit_exceeded' | 'redirect_loop' | 'proxy_connection_failed';
+    headers: Record<string, string | string[] | undefined>;
+    body: string;
+    redirectChain: RedirectStep[];
+    etag: string | null;
+    lastModified: string | null;
+    finalUrl: string;
+    retries?: number;
+    bytesReceived?: number;
+}
+export interface FetchOptions {
+    etag?: string;
+    lastModified?: string;
+    rate?: number;
+    maxBytes?: number;
+    crawlDelay?: number;
+}
+export declare class Fetcher {
+    private userAgent;
+    private rateLimiter;
+    private proxyAdapter;
+    private scopeManager?;
+    private maxRedirects;
+    constructor(options?: {
+        rate?: number;
+        proxyUrl?: string;
+        scopeManager?: ScopeManager;
+        maxRedirects?: number;
+        userAgent?: string;
+    });
+    fetch(url: string, options?: FetchOptions): Promise<FetchResult>;
+    private errorResult;
+}

package/dist/crawler/fetcher.js

@@ -0,0 +1,161 @@
+import { request } from 'undici';
+import { IPGuard } from '../core/security/ipGuard.js';
+import { RateLimiter } from '../core/network/rateLimiter.js';
+import { RetryPolicy } from '../core/network/retryPolicy.js';
+import { ResponseLimiter } from '../core/network/responseLimiter.js';
+import { RedirectController } from '../core/network/redirectController.js';
+import { ProxyAdapter } from '../core/network/proxyAdapter.js';
+import { version } from '../utils/version.js';
+export class Fetcher {
+    userAgent = 'crawlith/1.0';
+    rateLimiter;
+    proxyAdapter;
+    scopeManager;
+    maxRedirects;
+    constructor(options = {}) {
+        this.rateLimiter = new RateLimiter(options.rate || 2);
+        this.proxyAdapter = new ProxyAdapter(options.proxyUrl);
+        this.scopeManager = options.scopeManager;
+        this.maxRedirects = Math.min(options.maxRedirects ?? 2, 11);
+        this.userAgent = options.userAgent || `crawlith/${version}`;
+    }
+    async fetch(url, options = {}) {
+        const maxBytes = options.maxBytes || 2000000;
+        const redirectChain = [];
+        const redirectController = new RedirectController(this.maxRedirects, url);
+        let currentUrl = url;
+        let totalRetries = 0;
+        // Use a while(true) and explicit return/continue to handle redirects
+        while (true) {
+            const urlObj = new URL(currentUrl);
+            // 1. SSRF Guard
+            const isSafe = await IPGuard.validateHost(urlObj.hostname);
+            if (!isSafe) {
+                return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries);
+            }
+            // 2. Scope Validation (Domain & Subdomain)
+            if (this.scopeManager) {
+                const eligibility = this.scopeManager.isUrlEligible(currentUrl);
+                if (eligibility !== 'allowed') {
+                    return this.errorResult(eligibility, currentUrl, redirectChain, totalRetries);
+                }
+            }
+            // 3. Rate Limiting
+            await this.rateLimiter.waitForToken(urlObj.hostname, options.crawlDelay);
+            try {
+                // 4. Retry Strategy
+                const result = await RetryPolicy.execute(async (attempt) => {
+                    if (attempt > 0)
+                        totalRetries++;
+                    const headers = {
+                        'User-Agent': this.userAgent
+                    };
+                    // Conditional GET only for the FIRST request in a chain
+                    if (redirectChain.length === 0) {
+                        if (options.etag)
+                            headers['If-None-Match'] = options.etag;
+                        if (options.lastModified)
+                            headers['If-Modified-Since'] = options.lastModified;
+                    }
+                    const res = await request(currentUrl, {
+                        method: 'GET',
+                        headers,
+                        maxRedirections: 0,
+                        dispatcher: this.proxyAdapter.dispatcher,
+                        headersTimeout: 10000,
+                        bodyTimeout: 10000
+                    });
+                    if (RetryPolicy.isRetryableStatus(res.statusCode)) {
+                        await res.body.dump();
+                        throw new Error(`Status ${res.statusCode}`);
+                    }
+                    return res;
+                }, (error) => RetryPolicy.isNetworkError(error) || error.message.startsWith('Status '));
+                const status = result.statusCode;
+                const resHeaders = result.headers;
+                const getHeader = (name) => {
+                    const val = resHeaders[name.toLowerCase()];
+                    if (Array.isArray(val))
+                        return val[0];
+                    return val || null;
+                };
+                const etag = getHeader('etag');
+                const lastModified = getHeader('last-modified');
+                // Handle Redirects
+                if (status >= 300 && status < 400 && status !== 304) {
+                    const location = getHeader('location');
+                    if (location) {
+                        let targetUrl;
+                        try {
+                            targetUrl = new URL(location, currentUrl).toString();
+                        }
+                        catch (_e) {
+                            // Bad redirect location, treat as final but maybe error?
+                            const body = await ResponseLimiter.streamToString(result.body, maxBytes);
+                            return { status, headers: resHeaders, body, redirectChain, etag: null, lastModified: null, finalUrl: currentUrl, retries: totalRetries };
+                        }
+                        const redirectError = redirectController.nextHop(targetUrl);
+                        if (redirectError) {
+                            await result.body.dump();
+                            return this.errorResult(redirectError, currentUrl, redirectChain, totalRetries);
+                        }
+                        redirectChain.push({ url: currentUrl, status, target: targetUrl });
+                        await result.body.dump();
+                        currentUrl = targetUrl;
+                        continue; // Next iteration for redirect target
+                    }
+                }
+                // 5. Max Response Size (Streaming)
+                let bytesReceived = 0;
+                try {
+                    const body = status === 304 ? '' : await ResponseLimiter.streamToString(result.body, maxBytes, (bytes) => { bytesReceived = bytes; });
+                    return {
+                        status,
+                        headers: resHeaders,
+                        body,
+                        redirectChain,
+                        etag,
+                        lastModified,
+                        finalUrl: currentUrl,
+                        retries: totalRetries,
+                        bytesReceived
+                    };
+                }
+                catch (e) {
+                    if (e.message === 'Oversized response') {
+                        return {
+                            status: 'oversized',
+                            headers: resHeaders,
+                            body: '',
+                            redirectChain,
+                            etag: null,
+                            lastModified: null,
+                            finalUrl: currentUrl,
+                            retries: totalRetries,
+                            bytesReceived
+                        };
+                    }
+                    throw e;
+                }
+            }
+            catch (error) {
+                // Map common network errors to specific statuses if needed
+                const isProxyError = error.message?.toLowerCase().includes('proxy') || error.code === 'ECONNREFUSED';
+                const finalStatus = isProxyError ? 'proxy_connection_failed' : 'network_error';
+                return this.errorResult(totalRetries >= RetryPolicy.DEFAULT_CONFIG.maxRetries ? 'failed_after_retries' : finalStatus, currentUrl, redirectChain, totalRetries);
+            }
+        }
+    }
+    errorResult(status, finalUrl, redirectChain, retries) {
+        return {
+            status,
+            headers: {},
+            body: '',
+            redirectChain,
+            etag: null,
+            lastModified: null,
+            finalUrl,
+            retries
+        };
+    }
+}