@crawlith/core 0.1.0 → 0.1.2

package/src/audit/scoring.ts

@@ -1,232 +0,0 @@
-/* eslint-disable no-useless-assignment */
-import { TransportDiagnostics, DnsDiagnostics, SecurityHeadersResult, PerformanceMetrics, AuditIssue } from './types.js';
-
-interface CategoryScores {
-  transport: number;
-  security: number;
-  performance: number;
-  infrastructure: number;
-}
-
-export function calculateScore(
-  transport: TransportDiagnostics,
-  dns: DnsDiagnostics,
-  headers: SecurityHeadersResult,
-  performance: PerformanceMetrics,
-  existingIssues: AuditIssue[]
-): { score: number; grade: 'A' | 'B' | 'C' | 'D' | 'F'; issues: AuditIssue[]; categoryScores: CategoryScores } {
-
-  const issues: AuditIssue[] = [...existingIssues];
-  let transportScore = 0; // Max 30
-  let securityScore = 0;  // Max 20
-  let performanceScore = 0; // Max 30
-  let infrastructureScore = 0; // Max 20
-
-  // 1. Transport Security (30 pts)
-  // TLS Version
-  if (transport.tlsVersion) {
-    const version = parseFloat(transport.tlsVersion.replace('v', '').replace('TLS', '').trim());
-    if (version >= 1.2) {
-      transportScore += 15;
-    } else {
-      issues.push({
-        id: 'tls-old',
-        severity: 'severe',
-        category: 'tls',
-        message: `Deprecated TLS version: ${transport.tlsVersion}`,
-        scorePenalty: 15
-      });
-    }
-  } else if (transport.certificate) {
-    // HTTPS but no version detected? Unlikely.
-  } else {
-     // HTTP only?
-     issues.push({
-       id: 'no-https',
-       severity: 'critical',
-       category: 'tls',
-       message: 'Site is not using HTTPS',
-       scorePenalty: 30
-     });
-  }
-
-  // Certificate
-  if (transport.certificate) {
-    if (transport.certificate.isValidChain && !transport.certificate.isSelfSigned) {
-      transportScore += 15;
-    } else {
-      // Already caught in transport.ts, but let's ensure score reflects it
-      // If issues has cert-invalid, we don't add points.
-    }
-
-    if (transport.certificate.daysUntilExpiry < 30 && transport.certificate.daysUntilExpiry >= 0) {
-       issues.push({
-         id: 'cert-expiring-soon',
-         severity: 'moderate',
-         category: 'tls',
-         message: `Certificate expires in ${transport.certificate.daysUntilExpiry} days`,
-         scorePenalty: 5
-       });
-       // Penalty applied to transport score logic implicitly by not reaching max,
-       // but here we are adding up points.
-       // Let's deduct from the 15 points we might have given.
-       transportScore -= 5;
-    } else if (transport.certificate.daysUntilExpiry < 0) {
-       issues.push({
-         id: 'cert-expired',
-         severity: 'critical',
-         category: 'tls',
-         message: `Certificate expired on ${transport.certificate.validTo}`,
-         scorePenalty: 30
-       });
-       transportScore = 0; // Reset transport score
-    }
-  }
-
-  // 2. Response Security (Headers) (20 pts)
-  // headers.score is 0-100. Map to 0-20.
-  securityScore = (headers.score / 100) * 20;
-
-  // Add issues for missing critical headers
-  if (!headers.strictTransportSecurity.present) {
-    issues.push({
-      id: 'hsts-missing',
-      severity: 'moderate',
-      category: 'headers',
-      message: 'Missing Strict-Transport-Security header',
-      scorePenalty: 5
-    });
-  }
-  if (!headers.contentSecurityPolicy.present) {
-    issues.push({
-        id: 'csp-missing',
-        severity: 'moderate',
-        category: 'headers',
-        message: 'Missing Content-Security-Policy header',
-        scorePenalty: 5
-      });
-  }
-
-  // 3. Performance (30 pts)
-  // HTTP/2 (5 pts)
-  if (transport.alpnProtocol === 'h2' || transport.httpVersion === '2.0') {
-    performanceScore += 5;
-  } else {
-    issues.push({
-      id: 'no-h2',
-      severity: 'minor',
-      category: 'performance',
-      message: 'HTTP/2 not supported',
-      scorePenalty: 5
-    });
-  }
-
-  // Compression (5 pts)
-  if (transport.compression.length > 0) {
-    performanceScore += 5;
-  } else {
-    issues.push({
-      id: 'no-compression',
-      severity: 'moderate',
-      category: 'performance',
-      message: 'No compression enabled (gzip/br)',
-      scorePenalty: 5
-    });
-  }
-
-  // TTFB (10 pts)
-  if (performance.ttfb < 800) {
-    performanceScore += 10;
-  } else {
-    issues.push({
-      id: 'slow-ttfb',
-      severity: 'moderate',
-      category: 'performance',
-      message: `Slow TTFB: ${performance.ttfb.toFixed(0)}ms`,
-      scorePenalty: 10
-    });
-  }
-
-  // Redirects (5 pts)
-  if (transport.redirectCount <= 3) {
-    performanceScore += 5;
-  } else {
-    issues.push({
-      id: 'too-many-redirects',
-      severity: 'moderate',
-      category: 'performance',
-      message: `Too many redirects: ${transport.redirectCount}`,
-      scorePenalty: 5
-    });
-  }
-
-  // HTML Size (5 pts)
-  if (performance.htmlSize < 1024 * 1024) { // 1MB
-    performanceScore += 5;
-  } else {
-     issues.push({
-      id: 'large-html',
-      severity: 'minor',
-      category: 'performance',
-      message: `HTML size > 1MB: ${(performance.htmlSize / 1024 / 1024).toFixed(2)}MB`,
-      scorePenalty: 5
-    });
-  }
-
-  // 4. Infrastructure (20 pts)
-  // IPv6 (10 pts)
-  if (dns.ipv6Support) {
-    infrastructureScore += 10;
-  } else {
-    issues.push({
-      id: 'no-ipv6',
-      severity: 'minor',
-      category: 'dns',
-      message: 'No IPv6 DNS records found',
-      scorePenalty: 5
-    });
-  }
-
-  // Redundancy (10 pts)
-  if (dns.ipCount > 1) {
-    infrastructureScore += 10;
-  } else {
-    issues.push({
-      id: 'single-ip',
-      severity: 'minor',
-      category: 'dns',
-      message: 'Single IP address detected (no redundancy)',
-      scorePenalty: 5
-    });
-  }
-
-  let totalScore = transportScore + securityScore + performanceScore + infrastructureScore;
-
-  // Critical Overrides
-  const criticalIssues = issues.filter(i => i.severity === 'critical');
-  if (criticalIssues.length > 0) {
-    totalScore = Math.min(totalScore, 39); // Cap at F (<40)
-  }
-
-  const grade = getGrade(totalScore);
-
-  return {
-    score: Math.round(totalScore),
-    grade,
-    issues,
-    categoryScores: {
-        transport: transportScore,
-        security: securityScore,
-        performance: performanceScore,
-        infrastructure: infrastructureScore
-    }
-  };
-}
-
-function getGrade(score: number): 'A' | 'B' | 'C' | 'D' | 'F' {
-  if (score >= 90) return 'A';
-  if (score >= 75) return 'B';
-  if (score >= 60) return 'C';
-  if (score >= 40) return 'D';
-  return 'F';
-}

package/src/audit/transport.ts

@@ -1,258 +0,0 @@
-import https from 'node:https';
-import http from 'node:http';
-import tls from 'node:tls';
-import { URL } from 'node:url';
-import { IPGuard } from '../core/security/ipGuard.js';
-import { TransportDiagnostics, PerformanceMetrics, CertificateInfo, RedirectInfo, AuditIssue } from './types.js';
-import { IncomingMessage } from 'node:http';
-
-interface RequestResult {
-  url: string;
-  response: IncomingMessage;
-  body: Buffer;
-  timings: {
-    dns: number;
-    tcp: number;
-    tls: number;
-    ttfb: number;
-    total: number;
-  };
-  socket: any;
-  redirectUrl: string | null;
-}
-
-export async function analyzeTransport(targetUrl: string, timeout: number): Promise<{
-  transport: TransportDiagnostics;
-  performance: PerformanceMetrics;
-  issues: AuditIssue[];
-}> {
-  const maxRedirects = 10;
-  let currentUrl = targetUrl;
-  let redirectCount = 0;
-  const redirects: RedirectInfo[] = [];
-  const issues: AuditIssue[] = [];
-
-  // Cumulative metrics
-  let totalRedirectTime = 0;
-
-  for (let i = 0; i < maxRedirects; i++) {
-    const urlObj = new URL(currentUrl);
-    const isSafe = await IPGuard.validateHost(urlObj.hostname);
-    if (!isSafe) {
-      throw new Error(`Blocked: Redirect to internal/private IP prohibited (${currentUrl})`);
-    }
-
-    try {
-      const result = await executeRequest(currentUrl, timeout);
-
-      if (result.redirectUrl) {
-        redirectCount++;
-        totalRedirectTime += result.timings.total;
-
-        redirects.push({
-          url: currentUrl,
-          statusCode: result.response.statusCode || 0,
-          location: result.redirectUrl
-        });
-
-        currentUrl = result.redirectUrl;
-        continue;
-      }
-
-      // Final destination reached
-      const { response, body, timings, socket } = result;
-
-      // Collect Certificate Info
-      let certInfo: CertificateInfo | null = null;
-      let tlsVersion: string | null = null;
-      let cipherSuite: string | null = null;
-      let alpnProtocol: string | null = null;
-
-      if (socket instanceof tls.TLSSocket) {
-        const cert = socket.getPeerCertificate(true);
-        tlsVersion = socket.getProtocol();
-        const cipher = socket.getCipher();
-        cipherSuite = cipher ? cipher.name : null;
-        alpnProtocol = socket.alpnProtocol || null;
-
-        if (cert && Object.keys(cert).length > 0) {
-          certInfo = {
-            subject: (cert.subject && cert.subject.CN) ? cert.subject.CN : 'Unknown',
-            issuer: (cert.issuer && cert.issuer.CN) ? cert.issuer.CN : 'Unknown',
-            validFrom: cert.valid_from,
-            validTo: cert.valid_to,
-            daysUntilExpiry: Math.floor((new Date(cert.valid_to).getTime() - Date.now()) / (1000 * 60 * 60 * 24)),
-            isSelfSigned: cert.issuer && cert.subject && cert.issuer.CN === cert.subject.CN,
-            isValidChain: socket.authorized,
-            fingerprint: cert.fingerprint,
-            serialNumber: cert.serialNumber,
-            subjectAltName: cert.subjectaltname
-          };
-
-          if (!socket.authorized) {
-            issues.push({
-              id: 'cert-invalid',
-              severity: 'severe',
-              category: 'tls',
-              message: `Certificate validation failed: ${socket.authorizationError}`,
-              scorePenalty: 30
-            });
-          }
-        }
-      }
-
-      const httpVersion = response.httpVersion;
-      const contentEncoding = response.headers['content-encoding'];
-      const compression: string[] = [];
-      if (contentEncoding) {
-        compression.push(contentEncoding);
-      }
-
-      const connectionHeader = response.headers['connection'];
-      const keepAlive = connectionHeader ? connectionHeader.toLowerCase() !== 'close' : true;
-      const serverHeader = (response.headers['server'] as string) || null;
-
-      const headerText = `HTTP/${response.httpVersion} ${response.statusCode} ${response.statusMessage}\r\n` +
-        Object.entries(response.headers).map(([k, v]) => `${k}: ${v}`).join('\r\n') +
-        '\r\n\r\n';
-      const headerSize = Buffer.byteLength(headerText);
-      const htmlSize = body.length;
-
-      const transport: TransportDiagnostics = {
-        tlsVersion,
-        cipherSuite,
-        alpnProtocol: alpnProtocol || (httpVersion === '2.0' ? 'h2' : 'http/1.1'),
-        certificate: certInfo,
-        httpVersion,
-        compression,
-        keepAlive,
-        transferEncoding: (response.headers['transfer-encoding'] as string) || null,
-        redirectCount,
-        redirects,
-        serverHeader,
-        headers: response.headers
-      };
-
-      const performance: PerformanceMetrics = {
-        dnsLookupTime: timings.dns,
-        tcpConnectTime: timings.tcp,
-        tlsHandshakeTime: timings.tls,
-        ttfb: timings.ttfb,
-        totalTime: timings.total + totalRedirectTime,
-        htmlSize,
-        headerSize,
-        redirectTime: totalRedirectTime
-      };
-
-      return { transport, performance, issues };
-
-    } catch (error: any) {
-      throw new Error(`Transport analysis failed for ${currentUrl}: ${error.message}`, { cause: error });
-    }
-  }
-
-  throw new Error(`Too many redirects (limit: ${maxRedirects})`);
-}
-
-function executeRequest(urlStr: string, timeout: number): Promise<RequestResult> {
-  return new Promise((resolve, reject) => {
-    let url: URL;
-    try {
-      url = new URL(urlStr);
-    } catch (_e) {
-      return reject(new Error(`Invalid URL: ${urlStr}`));
-    }
-
-    const isHttps = url.protocol === 'https:';
-    const requestModule = isHttps ? https : http;
-
-    const timings = {
-      dns: 0,
-      tcp: 0,
-      tls: 0,
-      ttfb: 0,
-      total: 0
-    };
-
-    const t0 = performance.now();
-    let tDNS = t0;
-    let tTCP = t0;
-    let tTLS = t0;
-    let tReqSent = 0;
-
-    // We use agent: false to force new connection for accurate timing
-    const options = {
-      method: 'GET',
-      timeout,
-      rejectUnauthorized: false,
-      agent: false,
-      headers: {
-        'User-Agent': 'Crawlith/Audit',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-        'Accept-Encoding': 'gzip, deflate, br'
-      }
-    };
-
-    const req = requestModule.request(url, options, (res) => {
-      // TTFB: Time from request sent to first byte of headers received
-      timings.ttfb = performance.now() - (tReqSent || t0);
-
-      const chunks: Buffer[] = [];
-      res.on('data', (chunk) => chunks.push(chunk));
-      res.on('end', () => {
-        timings.total = performance.now() - t0;
-        const body = Buffer.concat(chunks);
-
-        let redirectUrl: string | null = null;
-        if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
-          try {
-            redirectUrl = new URL(res.headers.location, urlStr).toString();
-          } catch (_e) {
-            // Ignore invalid redirect
-          }
-        }
-
-        resolve({
-          url: urlStr,
-          response: res,
-          body,
-          timings,
-          socket: res.socket,
-          redirectUrl
-        });
-      });
-    });
-
-    req.on('socket', (socket) => {
-      socket.on('lookup', () => {
-        tDNS = performance.now();
-        timings.dns = tDNS - t0;
-      });
-      socket.on('connect', () => {
-        tTCP = performance.now();
-        if (timings.dns === 0 && tDNS === t0) {
-          // No lookup event
-          timings.dns = 0;
-          tDNS = t0;
-        }
-        timings.tcp = tTCP - tDNS;
-      });
-      socket.on('secureConnect', () => {
-        tTLS = performance.now();
-        timings.tls = tTLS - tTCP;
-      });
-    });
-
-    req.on('finish', () => {
-      tReqSent = performance.now();
-    });
-
-    req.on('error', (err) => reject(err));
-    req.on('timeout', () => {
-      req.destroy();
-      reject(new Error('Request timed out'));
-    });
-
-    req.end();
-  });
-}

package/src/audit/types.ts

@@ -1,102 +0,0 @@
-
-export interface AuditResult {
-  url: string;
-  transport: TransportDiagnostics;
-  securityHeaders: SecurityHeadersResult;
-  dns: DnsDiagnostics;
-  performance: PerformanceMetrics;
-  score: number;
-  grade: 'A' | 'B' | 'C' | 'D' | 'F';
-  issues: AuditIssue[];
-}
-
-export interface TransportDiagnostics {
-  // TLS / SSL
-  tlsVersion: string | null;
-  cipherSuite: string | null;
-  alpnProtocol: string | null; // http/1.1, h2
-  certificate: CertificateInfo | null;
-
-  // HTTP Protocol
-  httpVersion: string;
-  compression: string[]; // gzip, br, deflate
-  keepAlive: boolean;
-  transferEncoding: string | null;
-  redirectCount: number;
-  redirects: RedirectInfo[];
-  serverHeader: string | null;
-  headers: Record<string, string | string[] | undefined>;
-}
-
-export interface CertificateInfo {
-  issuer: string;
-  subject: string;
-  validFrom: string;
-  validTo: string;
-  daysUntilExpiry: number;
-  isSelfSigned: boolean;
-  isValidChain: boolean; // basic check, relying on node tls rejectUnauthorized: true result if possible, or manual check
-  fingerprint: string;
-  serialNumber: string;
-  subjectAltName?: string;
-}
-
-export interface RedirectInfo {
-  url: string;
-  statusCode: number;
-  location: string | null;
-}
-
-export interface SecurityHeadersResult {
-  strictTransportSecurity: HeaderStatus;
-  contentSecurityPolicy: HeaderStatus;
-  xFrameOptions: HeaderStatus;
-  xContentTypeOptions: HeaderStatus;
-  referrerPolicy: HeaderStatus;
-  permissionsPolicy: HeaderStatus;
-
-  details: Record<string, string>; // raw values
-  score: number; // partial score contribution (0-100 normalized for headers section)
-}
-
-export interface HeaderStatus {
-  present: boolean;
-  value: string | null;
-  valid: boolean; // simple syntax check
-  issues?: string[];
-}
-
-export interface DnsDiagnostics {
-  a: string[];
-  aaaa: string[];
-  cname: string[];
-  reverse: string[];
-  ipCount: number;
-  ipv6Support: boolean;
-  resolutionTime: number;
-}
-
-export interface PerformanceMetrics {
-  dnsLookupTime: number; // ms
-  tcpConnectTime: number; // ms
-  tlsHandshakeTime: number; // ms
-  ttfb: number; // ms
-  totalTime: number; // ms
-  htmlSize: number; // bytes
-  headerSize: number; // bytes
-  redirectTime?: number; // accumulated time spent in redirects
-}
-
-export interface AuditIssue {
-  id: string; // unique code for tests/filtering
-  severity: 'critical' | 'severe' | 'moderate' | 'minor' | 'info';
-  category: 'tls' | 'http' | 'headers' | 'dns' | 'performance';
-  message: string;
-  scorePenalty: number;
-}
-
-export interface AuditOptions {
-  timeout?: number;
-  verbose?: boolean;
-  debug?: boolean;
-}

package/src/core/network/proxyAdapter.ts

@@ -1,21 +0,0 @@
-import { ProxyAgent } from 'undici';
-
-export class ProxyAdapter {
-    private agent?: ProxyAgent;
-
-    constructor(proxyUrl?: string) {
-        if (proxyUrl) {
-            try {
-                // Validate URL
-                new URL(proxyUrl);
-                this.agent = new ProxyAgent(proxyUrl);
-            } catch {
-                throw new Error(`Invalid proxy URL: ${proxyUrl}`);
-            }
-        }
-    }
-
-    get dispatcher() {
-        return this.agent;
-    }
-}

package/src/core/network/rateLimiter.ts

@@ -1,39 +0,0 @@
-export class RateLimiter {
-    private buckets: Map<string, { tokens: number; lastRefill: number }> = new Map();
-    private rate: number; // tokens per second
-
-    constructor(rate: number = 2) {
-        this.rate = rate;
-    }
-
-    async waitForToken(host: string, crawlDelay: number = 0): Promise<void> {
-        const effectiveRate = crawlDelay > 0 ? Math.min(this.rate, 1 / crawlDelay) : this.rate;
-        const interval = 1000 / effectiveRate;
-
-        if (!this.buckets.has(host)) {
-            this.buckets.set(host, { tokens: this.rate - 1, lastRefill: Date.now() });
-            return;
-        }
-
-        const bucket = this.buckets.get(host)!;
-
-        while (true) {
-            const now = Date.now();
-            const elapsed = now - bucket.lastRefill;
-
-            if (elapsed > 0) {
-                const newTokens = elapsed / interval;
-                bucket.tokens = Math.min(this.rate, bucket.tokens + newTokens);
-                bucket.lastRefill = now;
-            }
-
-            if (bucket.tokens >= 1) {
-                bucket.tokens -= 1;
-                return;
-            }
-
-            const waitTime = Math.max(0, interval - (Date.now() - bucket.lastRefill));
-            await new Promise(resolve => setTimeout(resolve, waitTime));
-        }
-    }
-}

package/src/core/network/redirectController.ts

@@ -1,47 +0,0 @@
-export class RedirectController {
-    private maxHops: number;
-    private currentHops: number = 0;
-    private history: Set<string> = new Set();
-
-    constructor(maxHops: number = 5, seedUrl?: string) {
-        this.maxHops = maxHops;
-        if (seedUrl) {
-            this.history.add(this.normalize(seedUrl));
-        }
-    }
-
-    /**
-     * Records a hop and checks if it's within limits and not a loop.
-     * Returns null if allowed, or an error status string if blocked.
-     */
-    nextHop(url: string): 'redirect_limit_exceeded' | 'redirect_loop' | null {
-        // Normalize URL for loop detection (basic)
-        const normalized = this.normalize(url);
-
-        if (this.history.has(normalized)) {
-            return 'redirect_loop';
-        }
-
-        if (this.currentHops >= this.maxHops) {
-            return 'redirect_limit_exceeded';
-        }
-
-        this.history.add(normalized);
-        this.currentHops++;
-        return null;
-    }
-
-    get hops(): number {
-        return this.currentHops;
-    }
-
-    private normalize(url: string): string {
-        try {
-            const u = new URL(url);
-            u.hash = ''; // Ignore hash for loop detection
-            return u.toString();
-        } catch {
-            return url;
-        }
-    }
-}