@crawlith/core 0.1.0 → 0.1.2

package/src/core/network/responseLimiter.ts

@@ -1,34 +0,0 @@
-import { Readable } from 'stream';
-
-export class ResponseLimiter {
-    static async streamToString(
-        stream: Readable,
-        maxBytes: number,
-        onOversized?: (bytes: number) => void
-    ): Promise<string> {
-        return new Promise((resolve, reject) => {
-            let accumulated = 0;
-            const chunks: Buffer[] = [];
-
-            stream.on('data', (chunk: any) => {
-                const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-                accumulated += buffer.length;
-                if (accumulated > maxBytes) {
-                    stream.destroy();
-                    if (onOversized) onOversized(accumulated);
-                    reject(new Error('Oversized response'));
-                    return;
-                }
-                chunks.push(buffer);
-            });
-
-            stream.on('end', () => {
-                resolve(Buffer.concat(chunks).toString('utf-8'));
-            });
-
-            stream.on('error', (err) => {
-                reject(err);
-            });
-        });
-    }
-}

package/src/core/network/retryPolicy.ts

@@ -1,57 +0,0 @@
-export interface RetryConfig {
-    maxRetries: number;
-    baseDelay: number;
-}
-
-export class RetryPolicy {
-    static DEFAULT_CONFIG: RetryConfig = {
-        maxRetries: 3,
-        baseDelay: 500
-    };
-
-    static async execute<T>(
-        operation: (attempt: number) => Promise<T>,
-        isRetryable: (error: any) => boolean,
-        config: RetryConfig = RetryPolicy.DEFAULT_CONFIG
-    ): Promise<T> {
-        let lastError: any;
-
-        for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
-            try {
-                return await operation(attempt);
-            } catch (error) {
-                lastError = error;
-
-                if (attempt === config.maxRetries || !isRetryable(error)) {
-                    throw error;
-                }
-
-                const delay = config.baseDelay * Math.pow(2, attempt);
-                const jitter = delay * 0.1 * (Math.random() * 2 - 1);
-                const finalDelay = Math.max(0, delay + jitter);
-
-                await new Promise(resolve => setTimeout(resolve, finalDelay));
-            }
-        }
-
-        throw lastError;
-    }
-
-    static isRetryableStatus(status: number): boolean {
-        return status === 429 || (status >= 500 && status <= 599);
-    }
-
-    static isNetworkError(error: any): boolean {
-        const code = error?.code || error?.cause?.code;
-        return [
-            'ETIMEDOUT',
-            'ECONNRESET',
-            'EADDRINUSE',
-            'ECONNREFUSED',
-            'EPIPE',
-            'ENOTFOUND',
-            'ENETUNREACH',
-            'EAI_AGAIN'
-        ].includes(code);
-    }
-}

package/src/core/scope/domainFilter.ts

@@ -1,45 +0,0 @@
-export class DomainFilter {
-    private allowed: Set<string>;
-    private denied: Set<string>;
-
-    constructor(allowed: string[] = [], denied: string[] = []) {
-        this.allowed = new Set(allowed.map(d => this.normalize(d)));
-        this.denied = new Set(denied.map(d => this.normalize(d)));
-    }
-
-    /**
-     * Normalizes a hostname: lowercase, strip trailing dot.
-     * Note: We expect hostnames, not URLs.
-     */
-    private normalize(hostname: string): string {
-        let h = hostname.toLowerCase().trim();
-        if (h.endsWith('.')) {
-            h = h.slice(0, -1);
-        }
-        // Use URL to handle punycode and basic validation if possible
-        try {
-            // We wrap it in a dummy URL to let the browser/node logic normalize it
-            const url = new URL(`http://${h}`);
-            return url.hostname;
-        } catch {
-            return h;
-        }
-    }
-
-    isAllowed(hostname: string): boolean {
-        const normalized = this.normalize(hostname);
-
-        // 1. Deny list match -> Reject
-        if (this.denied.has(normalized)) {
-            return false;
-        }
-
-        // 2. Allow list not empty AND no match -> Reject
-        if (this.allowed.size > 0 && !this.allowed.has(normalized)) {
-            return false;
-        }
-
-        // 3. Otherwise -> Allow
-        return true;
-    }
-}

package/src/core/scope/scopeManager.ts

@@ -1,52 +0,0 @@
-import { DomainFilter } from './domainFilter.js';
-import { SubdomainPolicy } from './subdomainPolicy.js';
-
-export interface ScopeOptions {
-    allowedDomains?: string[];
-    deniedDomains?: string[];
-    includeSubdomains?: boolean;
-    rootUrl: string;
-}
-
-export type EligibilityResult = 'allowed' | 'blocked_by_domain_filter' | 'blocked_subdomain';
-
-export class ScopeManager {
-    private domainFilter: DomainFilter;
-    private subdomainPolicy: SubdomainPolicy;
-    private explicitAllowed: Set<string>;
-
-    constructor(options: ScopeOptions) {
-        this.domainFilter = new DomainFilter(options.allowedDomains, options.deniedDomains);
-        this.subdomainPolicy = new SubdomainPolicy(options.rootUrl, options.includeSubdomains);
-        this.explicitAllowed = new Set((options.allowedDomains || []).map(d => {
-            let h = d.toLowerCase().trim();
-            if (h.endsWith('.')) h = h.slice(0, -1);
-            return h;
-        }));
-    }
-
-    isUrlEligible(url: string): EligibilityResult {
-        let hostname: string;
-        try {
-            hostname = new URL(url).hostname.toLowerCase();
-            if (hostname.endsWith('.')) hostname = hostname.slice(0, -1);
-        } catch {
-            return 'blocked_by_domain_filter'; // Invalid URL is effectively blocked
-        }
-
-        if (!this.domainFilter.isAllowed(hostname)) {
-            return 'blocked_by_domain_filter';
-        }
-
-        // If explicit whitelist is used, and this domain is in it, allow it
-        if (this.explicitAllowed.has(hostname)) {
-            return 'allowed';
-        }
-
-        if (!this.subdomainPolicy.isAllowed(hostname)) {
-            return 'blocked_subdomain';
-        }
-
-        return 'allowed';
-    }
-}

package/src/core/scope/subdomainPolicy.ts

@@ -1,39 +0,0 @@
-export class SubdomainPolicy {
-    private rootHost: string;
-    private includeSubdomains: boolean;
-
-    constructor(rootUrl: string, includeSubdomains: boolean = false) {
-        try {
-            this.rootHost = new URL(rootUrl).hostname.toLowerCase();
-            if (this.rootHost.endsWith('.')) {
-                this.rootHost = this.rootHost.slice(0, -1);
-            }
-        } catch {
-            this.rootHost = '';
-        }
-        this.includeSubdomains = includeSubdomains;
-    }
-
-    isAllowed(hostname: string): boolean {
-        let target = hostname.toLowerCase().trim();
-        if (target.endsWith('.')) {
-            target = target.slice(0, -1);
-        }
-
-        // Exact match is always allowed if rootHost is set
-        if (target === this.rootHost) {
-            return true;
-        }
-
-        if (!this.includeSubdomains) {
-            return false;
-        }
-
-        // Label-based check for subdomains
-        // target must end with .rootHost
-        if (!target.endsWith(`.${this.rootHost}`)) {
-            return false;
-        }
-        return true;
-    }
-}

package/src/core/security/ipGuard.ts

@@ -1,92 +0,0 @@
-import * as dns from 'dns';
-import * as net from 'net';
-import { promisify } from 'util';
-
-const resolve4 = promisify(dns.resolve4);
-const resolve6 = promisify(dns.resolve6);
-
-export class IPGuard {
-    /**
-     * Checks if an IP address is internal/private
-     */
-    static isInternal(ip: string): boolean {
-        if (net.isIPv4(ip)) {
-            const parts = ip.split('.').map(Number);
-
-            // 127.0.0.0/8
-            if (parts[0] === 127) return true;
-
-            // 10.0.0.0/8
-            if (parts[0] === 10) return true;
-
-            // 192.168.0.0/16
-            if (parts[0] === 192 && parts[1] === 168) return true;
-
-            // 172.16.0.0 – 172.31.255.255
-            if (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) return true;
-
-            // 169.254.0.0/16
-            if (parts[0] === 169 && parts[1] === 254) return true;
-
-            // 0.0.0.0/8
-            if (parts[0] === 0) return true;
-
-            return false;
-        }
-
-        if (net.isIPv6(ip)) {
-            // Normalize IPv6
-            const expanded = IPGuard.expandIPv6(ip);
-
-            // ::1
-            if (expanded === '0000:0000:0000:0000:0000:0000:0000:0001') return true;
-
-            // fc00::/7 (Unique Local Address) -> fc or fd
-            const firstWord = parseInt(expanded.split(':')[0], 16);
-            if ((firstWord & 0xfe00) === 0xfc00) return true;
-
-            // fe80::/10 (Link Local)
-            if ((firstWord & 0xffc0) === 0xfe80) return true;
-
-            return false;
-        }
-
-        return true; // Unknown format, block it for safety
-    }
-
-    /**
-     * Resolves a hostname and validates all result IPs
-     */
-    static async validateHost(host: string): Promise<boolean> {
-        if (net.isIP(host)) {
-            return !IPGuard.isInternal(host);
-        }
-
-        try {
-            const res4 = await resolve4(host).catch(() => [] as string[]);
-            const res6 = await resolve6(host).catch(() => [] as string[]);
-            const ips = [...res4, ...res6];
-
-            if (ips.length === 0) return true; // Let the fetcher handle DNS failures
-
-            return ips.every(ip => !IPGuard.isInternal(ip));
-        } catch (_e) {
-            // If resolution fails drastically, we block for safety or let fetcher try
-            return false;
-        }
-    }
-
-    private static expandIPv6(ip: string): string {
-        if (ip === '::') return '0000:0000:0000:0000:0000:0000:0000:0000';
-        let full = ip;
-        if (ip.includes('::')) {
-            const parts = ip.split('::');
-            const left = parts[0].split(':').filter(x => x !== '');
-            const right = parts[1].split(':').filter(x => x !== '');
-            const missing = 8 - (left.length + right.length);
-            const middle = Array(missing).fill('0000');
-            full = [...left, ...middle, ...right].join(':');
-        }
-        return full.split(':').map(part => part.padStart(4, '0')).join(':');
-    }
-}