@crawlith/core 0.1.0

package/dist/audit/transport.js

@@ -0,0 +1,207 @@
+import https from 'node:https';
+import http from 'node:http';
+import tls from 'node:tls';
+import { URL } from 'node:url';
+import { IPGuard } from '../core/security/ipGuard.js';
+export async function analyzeTransport(targetUrl, timeout) {
+    const maxRedirects = 10;
+    let currentUrl = targetUrl;
+    let redirectCount = 0;
+    const redirects = [];
+    const issues = [];
+    // Cumulative metrics
+    let totalRedirectTime = 0;
+    for (let i = 0; i < maxRedirects; i++) {
+        const urlObj = new URL(currentUrl);
+        const isSafe = await IPGuard.validateHost(urlObj.hostname);
+        if (!isSafe) {
+            throw new Error(`Blocked: Redirect to internal/private IP prohibited (${currentUrl})`);
+        }
+        try {
+            const result = await executeRequest(currentUrl, timeout);
+            if (result.redirectUrl) {
+                redirectCount++;
+                totalRedirectTime += result.timings.total;
+                redirects.push({
+                    url: currentUrl,
+                    statusCode: result.response.statusCode || 0,
+                    location: result.redirectUrl
+                });
+                currentUrl = result.redirectUrl;
+                continue;
+            }
+            // Final destination reached
+            const { response, body, timings, socket } = result;
+            // Collect Certificate Info
+            let certInfo = null;
+            let tlsVersion = null;
+            let cipherSuite = null;
+            let alpnProtocol = null;
+            if (socket instanceof tls.TLSSocket) {
+                const cert = socket.getPeerCertificate(true);
+                tlsVersion = socket.getProtocol();
+                const cipher = socket.getCipher();
+                cipherSuite = cipher ? cipher.name : null;
+                alpnProtocol = socket.alpnProtocol || null;
+                if (cert && Object.keys(cert).length > 0) {
+                    certInfo = {
+                        subject: (cert.subject && cert.subject.CN) ? cert.subject.CN : 'Unknown',
+                        issuer: (cert.issuer && cert.issuer.CN) ? cert.issuer.CN : 'Unknown',
+                        validFrom: cert.valid_from,
+                        validTo: cert.valid_to,
+                        daysUntilExpiry: Math.floor((new Date(cert.valid_to).getTime() - Date.now()) / (1000 * 60 * 60 * 24)),
+                        isSelfSigned: cert.issuer && cert.subject && cert.issuer.CN === cert.subject.CN,
+                        isValidChain: socket.authorized,
+                        fingerprint: cert.fingerprint,
+                        serialNumber: cert.serialNumber,
+                        subjectAltName: cert.subjectaltname
+                    };
+                    if (!socket.authorized) {
+                        issues.push({
+                            id: 'cert-invalid',
+                            severity: 'severe',
+                            category: 'tls',
+                            message: `Certificate validation failed: ${socket.authorizationError}`,
+                            scorePenalty: 30
+                        });
+                    }
+                }
+            }
+            const httpVersion = response.httpVersion;
+            const contentEncoding = response.headers['content-encoding'];
+            const compression = [];
+            if (contentEncoding) {
+                compression.push(contentEncoding);
+            }
+            const connectionHeader = response.headers['connection'];
+            const keepAlive = connectionHeader ? connectionHeader.toLowerCase() !== 'close' : true;
+            const serverHeader = response.headers['server'] || null;
+            const headerText = `HTTP/${response.httpVersion} ${response.statusCode} ${response.statusMessage}\r\n` +
+                Object.entries(response.headers).map(([k, v]) => `${k}: ${v}`).join('\r\n') +
+                '\r\n\r\n';
+            const headerSize = Buffer.byteLength(headerText);
+            const htmlSize = body.length;
+            const transport = {
+                tlsVersion,
+                cipherSuite,
+                alpnProtocol: alpnProtocol || (httpVersion === '2.0' ? 'h2' : 'http/1.1'),
+                certificate: certInfo,
+                httpVersion,
+                compression,
+                keepAlive,
+                transferEncoding: response.headers['transfer-encoding'] || null,
+                redirectCount,
+                redirects,
+                serverHeader,
+                headers: response.headers
+            };
+            const performance = {
+                dnsLookupTime: timings.dns,
+                tcpConnectTime: timings.tcp,
+                tlsHandshakeTime: timings.tls,
+                ttfb: timings.ttfb,
+                totalTime: timings.total + totalRedirectTime,
+                htmlSize,
+                headerSize,
+                redirectTime: totalRedirectTime
+            };
+            return { transport, performance, issues };
+        }
+        catch (error) {
+            throw new Error(`Transport analysis failed for ${currentUrl}: ${error.message}`, { cause: error });
+        }
+    }
+    throw new Error(`Too many redirects (limit: ${maxRedirects})`);
+}
+function executeRequest(urlStr, timeout) {
+    return new Promise((resolve, reject) => {
+        let url;
+        try {
+            url = new URL(urlStr);
+        }
+        catch (_e) {
+            return reject(new Error(`Invalid URL: ${urlStr}`));
+        }
+        const isHttps = url.protocol === 'https:';
+        const requestModule = isHttps ? https : http;
+        const timings = {
+            dns: 0,
+            tcp: 0,
+            tls: 0,
+            ttfb: 0,
+            total: 0
+        };
+        const t0 = performance.now();
+        let tDNS = t0;
+        let tTCP = t0;
+        let tTLS = t0;
+        let tReqSent = 0;
+        // We use agent: false to force new connection for accurate timing
+        const options = {
+            method: 'GET',
+            timeout,
+            rejectUnauthorized: false,
+            agent: false,
+            headers: {
+                'User-Agent': 'Crawlith/Audit',
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                'Accept-Encoding': 'gzip, deflate, br'
+            }
+        };
+        const req = requestModule.request(url, options, (res) => {
+            // TTFB: Time from request sent to first byte of headers received
+            timings.ttfb = performance.now() - (tReqSent || t0);
+            const chunks = [];
+            res.on('data', (chunk) => chunks.push(chunk));
+            res.on('end', () => {
+                timings.total = performance.now() - t0;
+                const body = Buffer.concat(chunks);
+                let redirectUrl = null;
+                if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+                    try {
+                        redirectUrl = new URL(res.headers.location, urlStr).toString();
+                    }
+                    catch (_e) {
+                        // Ignore invalid redirect
+                    }
+                }
+                resolve({
+                    url: urlStr,
+                    response: res,
+                    body,
+                    timings,
+                    socket: res.socket,
+                    redirectUrl
+                });
+            });
+        });
+        req.on('socket', (socket) => {
+            socket.on('lookup', () => {
+                tDNS = performance.now();
+                timings.dns = tDNS - t0;
+            });
+            socket.on('connect', () => {
+                tTCP = performance.now();
+                if (timings.dns === 0 && tDNS === t0) {
+                    // No lookup event
+                    timings.dns = 0;
+                    tDNS = t0;
+                }
+                timings.tcp = tTCP - tDNS;
+            });
+            socket.on('secureConnect', () => {
+                tTLS = performance.now();
+                timings.tls = tTLS - tTCP;
+            });
+        });
+        req.on('finish', () => {
+            tReqSent = performance.now();
+        });
+        req.on('error', (err) => reject(err));
+        req.on('timeout', () => {
+            req.destroy();
+            reject(new Error('Request timed out'));
+        });
+        req.end();
+    });
+}

package/dist/audit/types.d.ts

@@ -0,0 +1,88 @@
+export interface AuditResult {
+    url: string;
+    transport: TransportDiagnostics;
+    securityHeaders: SecurityHeadersResult;
+    dns: DnsDiagnostics;
+    performance: PerformanceMetrics;
+    score: number;
+    grade: 'A' | 'B' | 'C' | 'D' | 'F';
+    issues: AuditIssue[];
+}
+export interface TransportDiagnostics {
+    tlsVersion: string | null;
+    cipherSuite: string | null;
+    alpnProtocol: string | null;
+    certificate: CertificateInfo | null;
+    httpVersion: string;
+    compression: string[];
+    keepAlive: boolean;
+    transferEncoding: string | null;
+    redirectCount: number;
+    redirects: RedirectInfo[];
+    serverHeader: string | null;
+    headers: Record<string, string | string[] | undefined>;
+}
+export interface CertificateInfo {
+    issuer: string;
+    subject: string;
+    validFrom: string;
+    validTo: string;
+    daysUntilExpiry: number;
+    isSelfSigned: boolean;
+    isValidChain: boolean;
+    fingerprint: string;
+    serialNumber: string;
+    subjectAltName?: string;
+}
+export interface RedirectInfo {
+    url: string;
+    statusCode: number;
+    location: string | null;
+}
+export interface SecurityHeadersResult {
+    strictTransportSecurity: HeaderStatus;
+    contentSecurityPolicy: HeaderStatus;
+    xFrameOptions: HeaderStatus;
+    xContentTypeOptions: HeaderStatus;
+    referrerPolicy: HeaderStatus;
+    permissionsPolicy: HeaderStatus;
+    details: Record<string, string>;
+    score: number;
+}
+export interface HeaderStatus {
+    present: boolean;
+    value: string | null;
+    valid: boolean;
+    issues?: string[];
+}
+export interface DnsDiagnostics {
+    a: string[];
+    aaaa: string[];
+    cname: string[];
+    reverse: string[];
+    ipCount: number;
+    ipv6Support: boolean;
+    resolutionTime: number;
+}
+export interface PerformanceMetrics {
+    dnsLookupTime: number;
+    tcpConnectTime: number;
+    tlsHandshakeTime: number;
+    ttfb: number;
+    totalTime: number;
+    htmlSize: number;
+    headerSize: number;
+    redirectTime?: number;
+}
+export interface AuditIssue {
+    id: string;
+    severity: 'critical' | 'severe' | 'moderate' | 'minor' | 'info';
+    category: 'tls' | 'http' | 'headers' | 'dns' | 'performance';
+    message: string;
+    scorePenalty: number;
+}
+export interface AuditOptions {
+    timeout?: number;
+    verbose?: boolean;
+    debug?: boolean;
+}

package/dist/audit/types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/network/proxyAdapter.d.ts

@@ -0,0 +1,6 @@
+import { ProxyAgent } from 'undici';
+export declare class ProxyAdapter {
+    private agent?;
+    constructor(proxyUrl?: string);
+    get dispatcher(): ProxyAgent | undefined;
+}

package/dist/core/network/proxyAdapter.js

@@ -0,0 +1,19 @@
+import { ProxyAgent } from 'undici';
+export class ProxyAdapter {
+    agent;
+    constructor(proxyUrl) {
+        if (proxyUrl) {
+            try {
+                // Validate URL
+                new URL(proxyUrl);
+                this.agent = new ProxyAgent(proxyUrl);
+            }
+            catch {
+                throw new Error(`Invalid proxy URL: ${proxyUrl}`);
+            }
+        }
+    }
+    get dispatcher() {
+        return this.agent;
+    }
+}

package/dist/core/network/rateLimiter.d.ts

@@ -0,0 +1,6 @@
+export declare class RateLimiter {
+    private buckets;
+    private rate;
+    constructor(rate?: number);
+    waitForToken(host: string, crawlDelay?: number): Promise<void>;
+}

package/dist/core/network/rateLimiter.js

@@ -0,0 +1,31 @@
+export class RateLimiter {
+    buckets = new Map();
+    rate; // tokens per second
+    constructor(rate = 2) {
+        this.rate = rate;
+    }
+    async waitForToken(host, crawlDelay = 0) {
+        const effectiveRate = crawlDelay > 0 ? Math.min(this.rate, 1 / crawlDelay) : this.rate;
+        const interval = 1000 / effectiveRate;
+        if (!this.buckets.has(host)) {
+            this.buckets.set(host, { tokens: this.rate - 1, lastRefill: Date.now() });
+            return;
+        }
+        const bucket = this.buckets.get(host);
+        while (true) {
+            const now = Date.now();
+            const elapsed = now - bucket.lastRefill;
+            if (elapsed > 0) {
+                const newTokens = elapsed / interval;
+                bucket.tokens = Math.min(this.rate, bucket.tokens + newTokens);
+                bucket.lastRefill = now;
+            }
+            if (bucket.tokens >= 1) {
+                bucket.tokens -= 1;
+                return;
+            }
+            const waitTime = Math.max(0, interval - (Date.now() - bucket.lastRefill));
+            await new Promise(resolve => setTimeout(resolve, waitTime));
+        }
+    }
+}

package/dist/core/network/redirectController.d.ts

@@ -0,0 +1,13 @@
+export declare class RedirectController {
+    private maxHops;
+    private currentHops;
+    private history;
+    constructor(maxHops?: number, seedUrl?: string);
+    /**
+     * Records a hop and checks if it's within limits and not a loop.
+     * Returns null if allowed, or an error status string if blocked.
+     */
+    nextHop(url: string): 'redirect_limit_exceeded' | 'redirect_loop' | null;
+    get hops(): number;
+    private normalize;
+}

package/dist/core/network/redirectController.js

@@ -0,0 +1,41 @@
+export class RedirectController {
+    maxHops;
+    currentHops = 0;
+    history = new Set();
+    constructor(maxHops = 5, seedUrl) {
+        this.maxHops = maxHops;
+        if (seedUrl) {
+            this.history.add(this.normalize(seedUrl));
+        }
+    }
+    /**
+     * Records a hop and checks if it's within limits and not a loop.
+     * Returns null if allowed, or an error status string if blocked.
+     */
+    nextHop(url) {
+        // Normalize URL for loop detection (basic)
+        const normalized = this.normalize(url);
+        if (this.history.has(normalized)) {
+            return 'redirect_loop';
+        }
+        if (this.currentHops >= this.maxHops) {
+            return 'redirect_limit_exceeded';
+        }
+        this.history.add(normalized);
+        this.currentHops++;
+        return null;
+    }
+    get hops() {
+        return this.currentHops;
+    }
+    normalize(url) {
+        try {
+            const u = new URL(url);
+            u.hash = ''; // Ignore hash for loop detection
+            return u.toString();
+        }
+        catch {
+            return url;
+        }
+    }
+}

package/dist/core/network/responseLimiter.d.ts

@@ -0,0 +1,4 @@
+import { Readable } from 'stream';
+export declare class ResponseLimiter {
+    static streamToString(stream: Readable, maxBytes: number, onOversized?: (bytes: number) => void): Promise<string>;
+}

package/dist/core/network/responseLimiter.js

@@ -0,0 +1,26 @@
+export class ResponseLimiter {
+    static async streamToString(stream, maxBytes, onOversized) {
+        return new Promise((resolve, reject) => {
+            let accumulated = 0;
+            const chunks = [];
+            stream.on('data', (chunk) => {
+                const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+                accumulated += buffer.length;
+                if (accumulated > maxBytes) {
+                    stream.destroy();
+                    if (onOversized)
+                        onOversized(accumulated);
+                    reject(new Error('Oversized response'));
+                    return;
+                }
+                chunks.push(buffer);
+            });
+            stream.on('end', () => {
+                resolve(Buffer.concat(chunks).toString('utf-8'));
+            });
+            stream.on('error', (err) => {
+                reject(err);
+            });
+        });
+    }
+}

package/dist/core/network/retryPolicy.d.ts

@@ -0,0 +1,10 @@
+export interface RetryConfig {
+    maxRetries: number;
+    baseDelay: number;
+}
+export declare class RetryPolicy {
+    static DEFAULT_CONFIG: RetryConfig;
+    static execute<T>(operation: (attempt: number) => Promise<T>, isRetryable: (error: any) => boolean, config?: RetryConfig): Promise<T>;
+    static isRetryableStatus(status: number): boolean;
+    static isNetworkError(error: any): boolean;
+}

package/dist/core/network/retryPolicy.js

@@ -0,0 +1,41 @@
+export class RetryPolicy {
+    static DEFAULT_CONFIG = {
+        maxRetries: 3,
+        baseDelay: 500
+    };
+    static async execute(operation, isRetryable, config = RetryPolicy.DEFAULT_CONFIG) {
+        let lastError;
+        for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
+            try {
+                return await operation(attempt);
+            }
+            catch (error) {
+                lastError = error;
+                if (attempt === config.maxRetries || !isRetryable(error)) {
+                    throw error;
+                }
+                const delay = config.baseDelay * Math.pow(2, attempt);
+                const jitter = delay * 0.1 * (Math.random() * 2 - 1);
+                const finalDelay = Math.max(0, delay + jitter);
+                await new Promise(resolve => setTimeout(resolve, finalDelay));
+            }
+        }
+        throw lastError;
+    }
+    static isRetryableStatus(status) {
+        return status === 429 || (status >= 500 && status <= 599);
+    }
+    static isNetworkError(error) {
+        const code = error?.code || error?.cause?.code;
+        return [
+            'ETIMEDOUT',
+            'ECONNRESET',
+            'EADDRINUSE',
+            'ECONNREFUSED',
+            'EPIPE',
+            'ENOTFOUND',
+            'ENETUNREACH',
+            'EAI_AGAIN'
+        ].includes(code);
+    }
+}

package/dist/core/scope/domainFilter.d.ts

@@ -0,0 +1,11 @@
+export declare class DomainFilter {
+    private allowed;
+    private denied;
+    constructor(allowed?: string[], denied?: string[]);
+    /**
+     * Normalizes a hostname: lowercase, strip trailing dot.
+     * Note: We expect hostnames, not URLs.
+     */
+    private normalize;
+    isAllowed(hostname: string): boolean;
+}

package/dist/core/scope/domainFilter.js

@@ -0,0 +1,40 @@
+export class DomainFilter {
+    allowed;
+    denied;
+    constructor(allowed = [], denied = []) {
+        this.allowed = new Set(allowed.map(d => this.normalize(d)));
+        this.denied = new Set(denied.map(d => this.normalize(d)));
+    }
+    /**
+     * Normalizes a hostname: lowercase, strip trailing dot.
+     * Note: We expect hostnames, not URLs.
+     */
+    normalize(hostname) {
+        let h = hostname.toLowerCase().trim();
+        if (h.endsWith('.')) {
+            h = h.slice(0, -1);
+        }
+        // Use URL to handle punycode and basic validation if possible
+        try {
+            // We wrap it in a dummy URL to let the browser/node logic normalize it
+            const url = new URL(`http://${h}`);
+            return url.hostname;
+        }
+        catch {
+            return h;
+        }
+    }
+    isAllowed(hostname) {
+        const normalized = this.normalize(hostname);
+        // 1. Deny list match -> Reject
+        if (this.denied.has(normalized)) {
+            return false;
+        }
+        // 2. Allow list not empty AND no match -> Reject
+        if (this.allowed.size > 0 && !this.allowed.has(normalized)) {
+            return false;
+        }
+        // 3. Otherwise -> Allow
+        return true;
+    }
+}

package/dist/core/scope/scopeManager.d.ts

@@ -0,0 +1,14 @@
+export interface ScopeOptions {
+    allowedDomains?: string[];
+    deniedDomains?: string[];
+    includeSubdomains?: boolean;
+    rootUrl: string;
+}
+export type EligibilityResult = 'allowed' | 'blocked_by_domain_filter' | 'blocked_subdomain';
+export declare class ScopeManager {
+    private domainFilter;
+    private subdomainPolicy;
+    private explicitAllowed;
+    constructor(options: ScopeOptions);
+    isUrlEligible(url: string): EligibilityResult;
+}

package/dist/core/scope/scopeManager.js

@@ -0,0 +1,39 @@
+import { DomainFilter } from './domainFilter.js';
+import { SubdomainPolicy } from './subdomainPolicy.js';
+export class ScopeManager {
+    domainFilter;
+    subdomainPolicy;
+    explicitAllowed;
+    constructor(options) {
+        this.domainFilter = new DomainFilter(options.allowedDomains, options.deniedDomains);
+        this.subdomainPolicy = new SubdomainPolicy(options.rootUrl, options.includeSubdomains);
+        this.explicitAllowed = new Set((options.allowedDomains || []).map(d => {
+            let h = d.toLowerCase().trim();
+            if (h.endsWith('.'))
+                h = h.slice(0, -1);
+            return h;
+        }));
+    }
+    isUrlEligible(url) {
+        let hostname;
+        try {
+            hostname = new URL(url).hostname.toLowerCase();
+            if (hostname.endsWith('.'))
+                hostname = hostname.slice(0, -1);
+        }
+        catch {
+            return 'blocked_by_domain_filter'; // Invalid URL is effectively blocked
+        }
+        if (!this.domainFilter.isAllowed(hostname)) {
+            return 'blocked_by_domain_filter';
+        }
+        // If explicit whitelist is used, and this domain is in it, allow it
+        if (this.explicitAllowed.has(hostname)) {
+            return 'allowed';
+        }
+        if (!this.subdomainPolicy.isAllowed(hostname)) {
+            return 'blocked_subdomain';
+        }
+        return 'allowed';
+    }
+}

package/dist/core/scope/subdomainPolicy.d.ts

@@ -0,0 +1,6 @@
+export declare class SubdomainPolicy {
+    private rootHost;
+    private includeSubdomains;
+    constructor(rootUrl: string, includeSubdomains?: boolean);
+    isAllowed(hostname: string): boolean;
+}

package/dist/core/scope/subdomainPolicy.js

@@ -0,0 +1,35 @@
+export class SubdomainPolicy {
+    rootHost;
+    includeSubdomains;
+    constructor(rootUrl, includeSubdomains = false) {
+        try {
+            this.rootHost = new URL(rootUrl).hostname.toLowerCase();
+            if (this.rootHost.endsWith('.')) {
+                this.rootHost = this.rootHost.slice(0, -1);
+            }
+        }
+        catch {
+            this.rootHost = '';
+        }
+        this.includeSubdomains = includeSubdomains;
+    }
+    isAllowed(hostname) {
+        let target = hostname.toLowerCase().trim();
+        if (target.endsWith('.')) {
+            target = target.slice(0, -1);
+        }
+        // Exact match is always allowed if rootHost is set
+        if (target === this.rootHost) {
+            return true;
+        }
+        if (!this.includeSubdomains) {
+            return false;
+        }
+        // Label-based check for subdomains
+        // target must end with .rootHost
+        if (!target.endsWith(`.${this.rootHost}`)) {
+            return false;
+        }
+        return true;
+    }
+}

package/dist/core/security/ipGuard.d.ts

@@ -0,0 +1,11 @@
+export declare class IPGuard {
+    /**
+     * Checks if an IP address is internal/private
+     */
+    static isInternal(ip: string): boolean;
+    /**
+     * Resolves a hostname and validates all result IPs
+     */
+    static validateHost(host: string): Promise<boolean>;
+    private static expandIPv6;
+}