@crawlith/core 0.1.1 → 0.1.2

package/src/audit/transport.ts

@@ -1,258 +0,0 @@
-import https from 'node:https';
-import http from 'node:http';
-import tls from 'node:tls';
-import { URL } from 'node:url';
-import { IPGuard } from '../core/security/ipGuard.js';
-import { TransportDiagnostics, PerformanceMetrics, CertificateInfo, RedirectInfo, AuditIssue } from './types.js';
-import { IncomingMessage } from 'node:http';
-
-interface RequestResult {
-  url: string;
-  response: IncomingMessage;
-  body: Buffer;
-  timings: {
-    dns: number;
-    tcp: number;
-    tls: number;
-    ttfb: number;
-    total: number;
-  };
-  socket: any;
-  redirectUrl: string | null;
-}
-
-export async function analyzeTransport(targetUrl: string, timeout: number): Promise<{
-  transport: TransportDiagnostics;
-  performance: PerformanceMetrics;
-  issues: AuditIssue[];
-}> {
-  const maxRedirects = 10;
-  let currentUrl = targetUrl;
-  let redirectCount = 0;
-  const redirects: RedirectInfo[] = [];
-  const issues: AuditIssue[] = [];
-
-  // Cumulative metrics
-  let totalRedirectTime = 0;
-
-  for (let i = 0; i < maxRedirects; i++) {
-    const urlObj = new URL(currentUrl);
-    const isSafe = await IPGuard.validateHost(urlObj.hostname);
-    if (!isSafe) {
-      throw new Error(`Blocked: Redirect to internal/private IP prohibited (${currentUrl})`);
-    }
-
-    try {
-      const result = await executeRequest(currentUrl, timeout);
-
-      if (result.redirectUrl) {
-        redirectCount++;
-        totalRedirectTime += result.timings.total;
-
-        redirects.push({
-          url: currentUrl,
-          statusCode: result.response.statusCode || 0,
-          location: result.redirectUrl
-        });
-
-        currentUrl = result.redirectUrl;
-        continue;
-      }
-
-      // Final destination reached
-      const { response, body, timings, socket } = result;
-
-      // Collect Certificate Info
-      let certInfo: CertificateInfo | null = null;
-      let tlsVersion: string | null = null;
-      let cipherSuite: string | null = null;
-      let alpnProtocol: string | null = null;
-
-      if (socket instanceof tls.TLSSocket) {
-        const cert = socket.getPeerCertificate(true);
-        tlsVersion = socket.getProtocol();
-        const cipher = socket.getCipher();
-        cipherSuite = cipher ? cipher.name : null;
-        alpnProtocol = socket.alpnProtocol || null;
-
-        if (cert && Object.keys(cert).length > 0) {
-          certInfo = {
-            subject: (cert.subject && cert.subject.CN) ? cert.subject.CN : 'Unknown',
-            issuer: (cert.issuer && cert.issuer.CN) ? cert.issuer.CN : 'Unknown',
-            validFrom: cert.valid_from,
-            validTo: cert.valid_to,
-            daysUntilExpiry: Math.floor((new Date(cert.valid_to).getTime() - Date.now()) / (1000 * 60 * 60 * 24)),
-            isSelfSigned: cert.issuer && cert.subject && cert.issuer.CN === cert.subject.CN,
-            isValidChain: socket.authorized,
-            fingerprint: cert.fingerprint,
-            serialNumber: cert.serialNumber,
-            subjectAltName: cert.subjectaltname
-          };
-
-          if (!socket.authorized) {
-            issues.push({
-              id: 'cert-invalid',
-              severity: 'severe',
-              category: 'tls',
-              message: `Certificate validation failed: ${socket.authorizationError}`,
-              scorePenalty: 30
-            });
-          }
-        }
-      }
-
-      const httpVersion = response.httpVersion;
-      const contentEncoding = response.headers['content-encoding'];
-      const compression: string[] = [];
-      if (contentEncoding) {
-        compression.push(contentEncoding);
-      }
-
-      const connectionHeader = response.headers['connection'];
-      const keepAlive = connectionHeader ? connectionHeader.toLowerCase() !== 'close' : true;
-      const serverHeader = (response.headers['server'] as string) || null;
-
-      const headerText = `HTTP/${response.httpVersion} ${response.statusCode} ${response.statusMessage}\r\n` +
-        Object.entries(response.headers).map(([k, v]) => `${k}: ${v}`).join('\r\n') +
-        '\r\n\r\n';
-      const headerSize = Buffer.byteLength(headerText);
-      const htmlSize = body.length;
-
-      const transport: TransportDiagnostics = {
-        tlsVersion,
-        cipherSuite,
-        alpnProtocol: alpnProtocol || (httpVersion === '2.0' ? 'h2' : 'http/1.1'),
-        certificate: certInfo,
-        httpVersion,
-        compression,
-        keepAlive,
-        transferEncoding: (response.headers['transfer-encoding'] as string) || null,
-        redirectCount,
-        redirects,
-        serverHeader,
-        headers: response.headers
-      };
-
-      const performance: PerformanceMetrics = {
-        dnsLookupTime: timings.dns,
-        tcpConnectTime: timings.tcp,
-        tlsHandshakeTime: timings.tls,
-        ttfb: timings.ttfb,
-        totalTime: timings.total + totalRedirectTime,
-        htmlSize,
-        headerSize,
-        redirectTime: totalRedirectTime
-      };
-
-      return { transport, performance, issues };
-
-    } catch (error: any) {
-      throw new Error(`Transport analysis failed for ${currentUrl}: ${error.message}`, { cause: error });
-    }
-  }
-
-  throw new Error(`Too many redirects (limit: ${maxRedirects})`);
-}
-
-function executeRequest(urlStr: string, timeout: number): Promise<RequestResult> {
-  return new Promise((resolve, reject) => {
-    let url: URL;
-    try {
-      url = new URL(urlStr);
-    } catch (_e) {
-      return reject(new Error(`Invalid URL: ${urlStr}`));
-    }
-
-    const isHttps = url.protocol === 'https:';
-    const requestModule = isHttps ? https : http;
-
-    const timings = {
-      dns: 0,
-      tcp: 0,
-      tls: 0,
-      ttfb: 0,
-      total: 0
-    };
-
-    const t0 = performance.now();
-    let tDNS = t0;
-    let tTCP = t0;
-    let tTLS = t0;
-    let tReqSent = 0;
-
-    // We use agent: false to force new connection for accurate timing
-    const options = {
-      method: 'GET',
-      timeout,
-      rejectUnauthorized: false,
-      agent: false,
-      headers: {
-        'User-Agent': 'Crawlith/Audit',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-        'Accept-Encoding': 'gzip, deflate, br'
-      }
-    };
-
-    const req = requestModule.request(url, options, (res) => {
-      // TTFB: Time from request sent to first byte of headers received
-      timings.ttfb = performance.now() - (tReqSent || t0);
-
-      const chunks: Buffer[] = [];
-      res.on('data', (chunk) => chunks.push(chunk));
-      res.on('end', () => {
-        timings.total = performance.now() - t0;
-        const body = Buffer.concat(chunks);
-
-        let redirectUrl: string | null = null;
-        if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
-          try {
-            redirectUrl = new URL(res.headers.location, urlStr).toString();
-          } catch (_e) {
-            // Ignore invalid redirect
-          }
-        }
-
-        resolve({
-          url: urlStr,
-          response: res,
-          body,
-          timings,
-          socket: res.socket,
-          redirectUrl
-        });
-      });
-    });
-
-    req.on('socket', (socket) => {
-      socket.on('lookup', () => {
-        tDNS = performance.now();
-        timings.dns = tDNS - t0;
-      });
-      socket.on('connect', () => {
-        tTCP = performance.now();
-        if (timings.dns === 0 && tDNS === t0) {
-          // No lookup event
-          timings.dns = 0;
-          tDNS = t0;
-        }
-        timings.tcp = tTCP - tDNS;
-      });
-      socket.on('secureConnect', () => {
-        tTLS = performance.now();
-        timings.tls = tTLS - tTCP;
-      });
-    });
-
-    req.on('finish', () => {
-      tReqSent = performance.now();
-    });
-
-    req.on('error', (err) => reject(err));
-    req.on('timeout', () => {
-      req.destroy();
-      reject(new Error('Request timed out'));
-    });
-
-    req.end();
-  });
-}

package/src/audit/types.ts

@@ -1,102 +0,0 @@
-
-export interface AuditResult {
-  url: string;
-  transport: TransportDiagnostics;
-  securityHeaders: SecurityHeadersResult;
-  dns: DnsDiagnostics;
-  performance: PerformanceMetrics;
-  score: number;
-  grade: 'A' | 'B' | 'C' | 'D' | 'F';
-  issues: AuditIssue[];
-}
-
-export interface TransportDiagnostics {
-  // TLS / SSL
-  tlsVersion: string | null;
-  cipherSuite: string | null;
-  alpnProtocol: string | null; // http/1.1, h2
-  certificate: CertificateInfo | null;
-
-  // HTTP Protocol
-  httpVersion: string;
-  compression: string[]; // gzip, br, deflate
-  keepAlive: boolean;
-  transferEncoding: string | null;
-  redirectCount: number;
-  redirects: RedirectInfo[];
-  serverHeader: string | null;
-  headers: Record<string, string | string[] | undefined>;
-}
-
-export interface CertificateInfo {
-  issuer: string;
-  subject: string;
-  validFrom: string;
-  validTo: string;
-  daysUntilExpiry: number;
-  isSelfSigned: boolean;
-  isValidChain: boolean; // basic check, relying on node tls rejectUnauthorized: true result if possible, or manual check
-  fingerprint: string;
-  serialNumber: string;
-  subjectAltName?: string;
-}
-
-export interface RedirectInfo {
-  url: string;
-  statusCode: number;
-  location: string | null;
-}
-
-export interface SecurityHeadersResult {
-  strictTransportSecurity: HeaderStatus;
-  contentSecurityPolicy: HeaderStatus;
-  xFrameOptions: HeaderStatus;
-  xContentTypeOptions: HeaderStatus;
-  referrerPolicy: HeaderStatus;
-  permissionsPolicy: HeaderStatus;
-
-  details: Record<string, string>; // raw values
-  score: number; // partial score contribution (0-100 normalized for headers section)
-}
-
-export interface HeaderStatus {
-  present: boolean;
-  value: string | null;
-  valid: boolean; // simple syntax check
-  issues?: string[];
-}
-
-export interface DnsDiagnostics {
-  a: string[];
-  aaaa: string[];
-  cname: string[];
-  reverse: string[];
-  ipCount: number;
-  ipv6Support: boolean;
-  resolutionTime: number;
-}
-
-export interface PerformanceMetrics {
-  dnsLookupTime: number; // ms
-  tcpConnectTime: number; // ms
-  tlsHandshakeTime: number; // ms
-  ttfb: number; // ms
-  totalTime: number; // ms
-  htmlSize: number; // bytes
-  headerSize: number; // bytes
-  redirectTime?: number; // accumulated time spent in redirects
-}
-
-export interface AuditIssue {
-  id: string; // unique code for tests/filtering
-  severity: 'critical' | 'severe' | 'moderate' | 'minor' | 'info';
-  category: 'tls' | 'http' | 'headers' | 'dns' | 'performance';
-  message: string;
-  scorePenalty: number;
-}
-
-export interface AuditOptions {
-  timeout?: number;
-  verbose?: boolean;
-  debug?: boolean;
-}

package/src/core/network/proxyAdapter.ts

@@ -1,21 +0,0 @@
-import { ProxyAgent } from 'undici';
-
-export class ProxyAdapter {
-    private agent?: ProxyAgent;
-
-    constructor(proxyUrl?: string) {
-        if (proxyUrl) {
-            try {
-                // Validate URL
-                new URL(proxyUrl);
-                this.agent = new ProxyAgent(proxyUrl);
-            } catch {
-                throw new Error(`Invalid proxy URL: ${proxyUrl}`);
-            }
-        }
-    }
-
-    get dispatcher() {
-        return this.agent;
-    }
-}

package/src/core/network/rateLimiter.ts

@@ -1,39 +0,0 @@
-export class RateLimiter {
-    private buckets: Map<string, { tokens: number; lastRefill: number }> = new Map();
-    private rate: number; // tokens per second
-
-    constructor(rate: number = 2) {
-        this.rate = rate;
-    }
-
-    async waitForToken(host: string, crawlDelay: number = 0): Promise<void> {
-        const effectiveRate = crawlDelay > 0 ? Math.min(this.rate, 1 / crawlDelay) : this.rate;
-        const interval = 1000 / effectiveRate;
-
-        if (!this.buckets.has(host)) {
-            this.buckets.set(host, { tokens: this.rate - 1, lastRefill: Date.now() });
-            return;
-        }
-
-        const bucket = this.buckets.get(host)!;
-
-        while (true) {
-            const now = Date.now();
-            const elapsed = now - bucket.lastRefill;
-
-            if (elapsed > 0) {
-                const newTokens = elapsed / interval;
-                bucket.tokens = Math.min(this.rate, bucket.tokens + newTokens);
-                bucket.lastRefill = now;
-            }
-
-            if (bucket.tokens >= 1) {
-                bucket.tokens -= 1;
-                return;
-            }
-
-            const waitTime = Math.max(0, interval - (Date.now() - bucket.lastRefill));
-            await new Promise(resolve => setTimeout(resolve, waitTime));
-        }
-    }
-}

package/src/core/network/redirectController.ts

@@ -1,47 +0,0 @@
-export class RedirectController {
-    private maxHops: number;
-    private currentHops: number = 0;
-    private history: Set<string> = new Set();
-
-    constructor(maxHops: number = 5, seedUrl?: string) {
-        this.maxHops = maxHops;
-        if (seedUrl) {
-            this.history.add(this.normalize(seedUrl));
-        }
-    }
-
-    /**
-     * Records a hop and checks if it's within limits and not a loop.
-     * Returns null if allowed, or an error status string if blocked.
-     */
-    nextHop(url: string): 'redirect_limit_exceeded' | 'redirect_loop' | null {
-        // Normalize URL for loop detection (basic)
-        const normalized = this.normalize(url);
-
-        if (this.history.has(normalized)) {
-            return 'redirect_loop';
-        }
-
-        if (this.currentHops >= this.maxHops) {
-            return 'redirect_limit_exceeded';
-        }
-
-        this.history.add(normalized);
-        this.currentHops++;
-        return null;
-    }
-
-    get hops(): number {
-        return this.currentHops;
-    }
-
-    private normalize(url: string): string {
-        try {
-            const u = new URL(url);
-            u.hash = ''; // Ignore hash for loop detection
-            return u.toString();
-        } catch {
-            return url;
-        }
-    }
-}

package/src/core/network/responseLimiter.ts

@@ -1,34 +0,0 @@
-import { Readable } from 'stream';
-
-export class ResponseLimiter {
-    static async streamToString(
-        stream: Readable,
-        maxBytes: number,
-        onOversized?: (bytes: number) => void
-    ): Promise<string> {
-        return new Promise((resolve, reject) => {
-            let accumulated = 0;
-            const chunks: Buffer[] = [];
-
-            stream.on('data', (chunk: any) => {
-                const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-                accumulated += buffer.length;
-                if (accumulated > maxBytes) {
-                    stream.destroy();
-                    if (onOversized) onOversized(accumulated);
-                    reject(new Error('Oversized response'));
-                    return;
-                }
-                chunks.push(buffer);
-            });
-
-            stream.on('end', () => {
-                resolve(Buffer.concat(chunks).toString('utf-8'));
-            });
-
-            stream.on('error', (err) => {
-                reject(err);
-            });
-        });
-    }
-}

package/src/core/network/retryPolicy.ts

@@ -1,57 +0,0 @@
-export interface RetryConfig {
-    maxRetries: number;
-    baseDelay: number;
-}
-
-export class RetryPolicy {
-    static DEFAULT_CONFIG: RetryConfig = {
-        maxRetries: 3,
-        baseDelay: 500
-    };
-
-    static async execute<T>(
-        operation: (attempt: number) => Promise<T>,
-        isRetryable: (error: any) => boolean,
-        config: RetryConfig = RetryPolicy.DEFAULT_CONFIG
-    ): Promise<T> {
-        let lastError: any;
-
-        for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
-            try {
-                return await operation(attempt);
-            } catch (error) {
-                lastError = error;
-
-                if (attempt === config.maxRetries || !isRetryable(error)) {
-                    throw error;
-                }
-
-                const delay = config.baseDelay * Math.pow(2, attempt);
-                const jitter = delay * 0.1 * (Math.random() * 2 - 1);
-                const finalDelay = Math.max(0, delay + jitter);
-
-                await new Promise(resolve => setTimeout(resolve, finalDelay));
-            }
-        }
-
-        throw lastError;
-    }
-
-    static isRetryableStatus(status: number): boolean {
-        return status === 429 || (status >= 500 && status <= 599);
-    }
-
-    static isNetworkError(error: any): boolean {
-        const code = error?.code || error?.cause?.code;
-        return [
-            'ETIMEDOUT',
-            'ECONNRESET',
-            'EADDRINUSE',
-            'ECONNREFUSED',
-            'EPIPE',
-            'ENOTFOUND',
-            'ENETUNREACH',
-            'EAI_AGAIN'
-        ].includes(code);
-    }
-}

package/src/core/scope/domainFilter.ts

@@ -1,45 +0,0 @@
-export class DomainFilter {
-    private allowed: Set<string>;
-    private denied: Set<string>;
-
-    constructor(allowed: string[] = [], denied: string[] = []) {
-        this.allowed = new Set(allowed.map(d => this.normalize(d)));
-        this.denied = new Set(denied.map(d => this.normalize(d)));
-    }
-
-    /**
-     * Normalizes a hostname: lowercase, strip trailing dot.
-     * Note: We expect hostnames, not URLs.
-     */
-    private normalize(hostname: string): string {
-        let h = hostname.toLowerCase().trim();
-        if (h.endsWith('.')) {
-            h = h.slice(0, -1);
-        }
-        // Use URL to handle punycode and basic validation if possible
-        try {
-            // We wrap it in a dummy URL to let the browser/node logic normalize it
-            const url = new URL(`http://${h}`);
-            return url.hostname;
-        } catch {
-            return h;
-        }
-    }
-
-    isAllowed(hostname: string): boolean {
-        const normalized = this.normalize(hostname);
-
-        // 1. Deny list match -> Reject
-        if (this.denied.has(normalized)) {
-            return false;
-        }
-
-        // 2. Allow list not empty AND no match -> Reject
-        if (this.allowed.size > 0 && !this.allowed.has(normalized)) {
-            return false;
-        }
-
-        // 3. Otherwise -> Allow
-        return true;
-    }
-}

package/src/core/scope/scopeManager.ts

@@ -1,52 +0,0 @@
-import { DomainFilter } from './domainFilter.js';
-import { SubdomainPolicy } from './subdomainPolicy.js';
-
-export interface ScopeOptions {
-    allowedDomains?: string[];
-    deniedDomains?: string[];
-    includeSubdomains?: boolean;
-    rootUrl: string;
-}
-
-export type EligibilityResult = 'allowed' | 'blocked_by_domain_filter' | 'blocked_subdomain';
-
-export class ScopeManager {
-    private domainFilter: DomainFilter;
-    private subdomainPolicy: SubdomainPolicy;
-    private explicitAllowed: Set<string>;
-
-    constructor(options: ScopeOptions) {
-        this.domainFilter = new DomainFilter(options.allowedDomains, options.deniedDomains);
-        this.subdomainPolicy = new SubdomainPolicy(options.rootUrl, options.includeSubdomains);
-        this.explicitAllowed = new Set((options.allowedDomains || []).map(d => {
-            let h = d.toLowerCase().trim();
-            if (h.endsWith('.')) h = h.slice(0, -1);
-            return h;
-        }));
-    }
-
-    isUrlEligible(url: string): EligibilityResult {
-        let hostname: string;
-        try {
-            hostname = new URL(url).hostname.toLowerCase();
-            if (hostname.endsWith('.')) hostname = hostname.slice(0, -1);
-        } catch {
-            return 'blocked_by_domain_filter'; // Invalid URL is effectively blocked
-        }
-
-        if (!this.domainFilter.isAllowed(hostname)) {
-            return 'blocked_by_domain_filter';
-        }
-
-        // If explicit whitelist is used, and this domain is in it, allow it
-        if (this.explicitAllowed.has(hostname)) {
-            return 'allowed';
-        }
-
-        if (!this.subdomainPolicy.isAllowed(hostname)) {
-            return 'blocked_subdomain';
-        }
-
-        return 'allowed';
-    }
-}

package/src/core/scope/subdomainPolicy.ts

@@ -1,39 +0,0 @@
-export class SubdomainPolicy {
-    private rootHost: string;
-    private includeSubdomains: boolean;
-
-    constructor(rootUrl: string, includeSubdomains: boolean = false) {
-        try {
-            this.rootHost = new URL(rootUrl).hostname.toLowerCase();
-            if (this.rootHost.endsWith('.')) {
-                this.rootHost = this.rootHost.slice(0, -1);
-            }
-        } catch {
-            this.rootHost = '';
-        }
-        this.includeSubdomains = includeSubdomains;
-    }
-
-    isAllowed(hostname: string): boolean {
-        let target = hostname.toLowerCase().trim();
-        if (target.endsWith('.')) {
-            target = target.slice(0, -1);
-        }
-
-        // Exact match is always allowed if rootHost is set
-        if (target === this.rootHost) {
-            return true;
-        }
-
-        if (!this.includeSubdomains) {
-            return false;
-        }
-
-        // Label-based check for subdomains
-        // target must end with .rootHost
-        if (!target.endsWith(`.${this.rootHost}`)) {
-            return false;
-        }
-        return true;
-    }
-}