@crawlith/core 0.1.0

package/src/analysis/scoring.ts

@@ -0,0 +1,59 @@
+import { Metrics } from '../graph/metrics.js';
+import type { PageAnalysis } from './analyze.js';
+
+export interface SiteScore {
+  seoHealthScore: number;
+  authorityEntropyOrphanScore: number;
+  overallScore: number;
+}
+
+export function scorePageSeo(page: PageAnalysis): number {
+  const titleMeta = (scoreTextStatus(page.title.status) + scoreTextStatus(page.metaDescription.status)) / 2;
+  const h1 = page.h1.status === 'ok' ? 100 : page.h1.status === 'warning' ? 60 : 10;
+  const wordQuality = Math.min(100, (page.content.wordCount / 600) * 100) * 0.7 + Math.min(100, page.content.textHtmlRatio * 500) * 0.3;
+  const thin = 100 - page.thinScore;
+  const imageDen = Math.max(1, page.images.totalImages);
+  const imageAlt = Math.max(0, 100 - ((page.images.missingAlt + page.images.emptyAlt) / imageDen) * 100);
+  const structured = page.structuredData.present ? (page.structuredData.valid ? 100 : 40) : 30;
+  const linkBalance = Math.max(0, 100 - Math.abs(page.links.externalRatio - 0.3) * 200);
+
+  const score =
+    titleMeta * 0.15 +
+    h1 * 0.1 +
+    wordQuality * 0.2 +
+    thin * 0.2 +
+    imageAlt * 0.1 +
+    structured * 0.1 +
+    linkBalance * 0.15;
+
+  return Number(Math.max(0, Math.min(100, score)).toFixed(2));
+}
+
+function scoreTextStatus(status: PageAnalysis['title']['status']): number {
+  switch (status) {
+    case 'ok': return 100;
+    case 'duplicate': return 45;
+    case 'too_short': return 60;
+    case 'too_long': return 60;
+    case 'missing': return 0;
+  }
+}
+
+export function aggregateSiteScore(metrics: Metrics, pages: PageAnalysis[]): SiteScore {
+  const seoHealthScore = pages.length === 0 ? 0 : pages.reduce((acc, page) => acc + page.seoScore, 0) / pages.length;
+
+  const avgAuthority = metrics.topAuthorityPages.length === 0
+    ? 0
+    : metrics.topAuthorityPages.reduce((acc, item) => acc + item.authority, 0) / metrics.topAuthorityPages.length;
+  const entropyScore = Math.max(0, 100 - Math.abs(metrics.structuralEntropy - 2) * 25);
+  const orphanPenalty = metrics.totalPages === 0 ? 0 : (metrics.orphanPages.length / metrics.totalPages) * 100;
+  const authorityEntropyOrphanScore = Math.max(0, Math.min(100, (avgAuthority * 100 * 0.4) + (entropyScore * 0.35) + ((100 - orphanPenalty) * 0.25)));
+
+  const overallScore = Number((seoHealthScore * 0.7 + authorityEntropyOrphanScore * 0.3).toFixed(2));
+
+  return {
+    seoHealthScore: Number(seoHealthScore.toFixed(2)),
+    authorityEntropyOrphanScore: Number(authorityEntropyOrphanScore.toFixed(2)),
+    overallScore
+  };
+}

package/src/analysis/seo.ts

@@ -0,0 +1,82 @@
+import { load } from 'cheerio';
+
+export type SeoStatus = 'ok' | 'missing' | 'too_short' | 'too_long' | 'duplicate';
+
+export interface TextFieldAnalysis {
+  value: string | null;
+  length: number;
+  status: SeoStatus;
+}
+
+export interface H1Analysis {
+  count: number;
+  status: 'ok' | 'critical' | 'warning';
+  matchesTitle: boolean;
+}
+
+function normalizedText(value: string | null): string {
+  return (value ?? '').trim().toLowerCase();
+}
+
+export function analyzeTitle(html: string): TextFieldAnalysis {
+  const $ = load(html);
+  const title = $('title').first().text().trim();
+  if (!title) {
+    return { value: null, length: 0, status: 'missing' };
+  }
+
+  if (title.length < 50) return { value: title, length: title.length, status: 'too_short' };
+  if (title.length > 60) return { value: title, length: title.length, status: 'too_long' };
+  return { value: title, length: title.length, status: 'ok' };
+}
+
+export function analyzeMetaDescription(html: string): TextFieldAnalysis {
+  const $ = load(html);
+  const raw = $('meta[name="description"]').attr('content');
+  if (raw === undefined) {
+    return { value: null, length: 0, status: 'missing' };
+  }
+
+  const description = raw.trim();
+  if (!description) {
+    return { value: '', length: 0, status: 'missing' };
+  }
+
+  if (description.length < 140) return { value: description, length: description.length, status: 'too_short' };
+  if (description.length > 160) return { value: description, length: description.length, status: 'too_long' };
+  return { value: description, length: description.length, status: 'ok' };
+}
+
+export function applyDuplicateStatuses<T extends TextFieldAnalysis>(fields: T[]): T[] {
+  const counts = new Map<string, number>();
+  for (const field of fields) {
+    const key = normalizedText(field.value);
+    if (!key) continue;
+    counts.set(key, (counts.get(key) || 0) + 1);
+  }
+
+  return fields.map((field) => {
+    const key = normalizedText(field.value);
+    if (!key) return field;
+    if ((counts.get(key) || 0) > 1) {
+      return { ...field, status: 'duplicate' };
+    }
+    return field;
+  });
+}
+
+export function analyzeH1(html: string, titleValue: string | null): H1Analysis {
+  const $ = load(html);
+  const h1Values = $('h1').toArray().map((el) => $(el).text().trim()).filter(Boolean);
+  const count = h1Values.length;
+  const first = h1Values[0] || null;
+  const matchesTitle = Boolean(first && titleValue && normalizedText(first) === normalizedText(titleValue));
+
+  if (count === 0) {
+    return { count, status: 'critical', matchesTitle };
+  }
+  if (count > 1) {
+    return { count, status: 'warning', matchesTitle };
+  }
+  return { count, status: 'ok', matchesTitle };
+}

package/src/analysis/structuredData.ts

@@ -0,0 +1,62 @@
+import { load } from 'cheerio';
+
+export interface StructuredDataResult {
+  present: boolean;
+  types: string[];
+  valid: boolean;
+}
+
+export function analyzeStructuredData(html: string): StructuredDataResult {
+  const $ = load(html);
+  const scripts = $('script[type="application/ld+json"]').toArray();
+  if (scripts.length === 0) {
+    return { present: false, types: [], valid: false };
+  }
+
+  const types = new Set<string>();
+  let valid = true;
+
+  for (const script of scripts) {
+    const raw = $(script).text().trim();
+    if (!raw) {
+      valid = false;
+      continue;
+    }
+
+    try {
+      const parsed = JSON.parse(raw);
+      extractTypes(parsed, types);
+    } catch {
+      valid = false;
+    }
+  }
+
+  return {
+    present: true,
+    valid,
+    types: Array.from(types)
+  };
+}
+
+function extractTypes(input: unknown, types: Set<string>): void {
+  if (Array.isArray(input)) {
+    input.forEach((item) => extractTypes(item, types));
+    return;
+  }
+
+  if (!input || typeof input !== 'object') return;
+
+  const maybeType = (input as Record<string, unknown>)['@type'];
+  if (typeof maybeType === 'string') {
+    types.add(maybeType);
+  } else if (Array.isArray(maybeType)) {
+    for (const item of maybeType) {
+      if (typeof item === 'string') types.add(item);
+    }
+  }
+
+  const graph = (input as Record<string, unknown>)['@graph'];
+  if (Array.isArray(graph)) {
+    graph.forEach((item) => extractTypes(item, types));
+  }
+}

package/src/audit/dns.ts

@@ -0,0 +1,49 @@
+import dns from 'node:dns/promises';
+import { DnsDiagnostics } from './types.js';
+
+export async function resolveDns(hostname: string): Promise<DnsDiagnostics> {
+  const start = performance.now();
+
+  const result: DnsDiagnostics = {
+    a: [],
+    aaaa: [],
+    cname: [],
+    reverse: [],
+    ipCount: 0,
+    ipv6Support: false,
+    resolutionTime: 0
+  };
+
+  try {
+    // We run these in parallel
+    const [a, aaaa, cname] = await Promise.all([
+      dns.resolve4(hostname).catch(() => [] as string[]),
+      dns.resolve6(hostname).catch(() => [] as string[]),
+      dns.resolveCname(hostname).catch(() => [] as string[])
+    ]);
+
+    result.a = a;
+    result.aaaa = aaaa;
+    result.cname = cname;
+    result.ipCount = a.length + aaaa.length;
+    result.ipv6Support = aaaa.length > 0;
+
+    // Try reverse lookup on first IP if available
+    const ipToReverse = a.length > 0 ? a[0] : (aaaa.length > 0 ? aaaa[0] : null);
+
+    if (ipToReverse) {
+      try {
+        result.reverse = await dns.reverse(ipToReverse);
+      } catch {
+        // Reverse lookup failed, ignore
+      }
+    }
+
+  } catch (_error) {
+    // DNS resolution failed entirely or other error
+    // We return empty result but with time measured
+  }
+
+  result.resolutionTime = performance.now() - start;
+  return result;
+}

package/src/audit/headers.ts

@@ -0,0 +1,98 @@
+import { SecurityHeadersResult, HeaderStatus } from './types.js';
+
+export function analyzeHeaders(headers: Record<string, string | string[] | undefined>): SecurityHeadersResult {
+  const normalized: Record<string, string> = {};
+  for (const [key, value] of Object.entries(headers)) {
+    if (typeof value === 'string') {
+      normalized[key.toLowerCase()] = value;
+    } else if (Array.isArray(value)) {
+      normalized[key.toLowerCase()] = value.join(', ');
+    }
+  }
+
+  const result: SecurityHeadersResult = {
+    strictTransportSecurity: checkHSTS(normalized['strict-transport-security']),
+    contentSecurityPolicy: checkCSP(normalized['content-security-policy']),
+    xFrameOptions: checkXFrameOptions(normalized['x-frame-options']),
+    xContentTypeOptions: checkXContentTypeOptions(normalized['x-content-type-options']),
+    referrerPolicy: checkReferrerPolicy(normalized['referrer-policy']),
+    permissionsPolicy: checkPermissionsPolicy(normalized['permissions-policy']),
+    details: normalized,
+    score: 0
+  };
+
+  // Calculate internal score (0-100) based on presence and validity
+  let score = 0;
+  const weights = {
+    hsts: 30,
+    csp: 25,
+    xframe: 15,
+    xcontent: 15,
+    referrer: 10,
+    permissions: 5
+  };
+
+  if (result.strictTransportSecurity.valid) score += weights.hsts;
+  if (result.contentSecurityPolicy.valid) score += weights.csp;
+  if (result.xFrameOptions.valid) score += weights.xframe;
+  if (result.xContentTypeOptions.valid) score += weights.xcontent;
+  if (result.referrerPolicy.valid) score += weights.referrer;
+  if (result.permissionsPolicy.valid) score += weights.permissions;
+
+  result.score = score;
+
+  return result;
+}
+
+function checkHSTS(value: string | undefined): HeaderStatus {
+  if (!value) return { present: false, value: null, valid: false, issues: ['Missing HSTS header'] };
+
+  const valid = value.includes('max-age=');
+  const issues: string[] = [];
+  if (!valid) issues.push('Missing max-age directive');
+  if (!value.includes('includeSubDomains')) issues.push('Missing includeSubDomains');
+
+  return { present: true, value, valid, issues };
+}
+
+function checkCSP(value: string | undefined): HeaderStatus {
+  if (!value) return { present: false, value: null, valid: false, issues: ['Missing CSP header'] };
+
+  // Basic check: non-empty
+  return { present: true, value, valid: value.length > 0, issues: [] };
+}
+
+function checkXFrameOptions(value: string | undefined): HeaderStatus {
+  if (!value) return { present: false, value: null, valid: false, issues: ['Missing X-Frame-Options'] };
+
+  const upper = value.toUpperCase();
+  const valid = upper === 'DENY' || upper === 'SAMEORIGIN';
+  return {
+    present: true,
+    value,
+    valid,
+    issues: valid ? [] : [`Invalid value: ${value}`]
+  };
+}
+
+function checkXContentTypeOptions(value: string | undefined): HeaderStatus {
+  if (!value) return { present: false, value: null, valid: false, issues: ['Missing X-Content-Type-Options'] };
+
+  const valid = value.toLowerCase() === 'nosniff';
+  return {
+    present: true,
+    value,
+    valid,
+    issues: valid ? [] : [`Invalid value: ${value}`]
+  };
+}
+
+function checkReferrerPolicy(value: string | undefined): HeaderStatus {
+  if (!value) return { present: false, value: null, valid: false, issues: ['Missing Referrer-Policy'] };
+  return { present: true, value, valid: true, issues: [] };
+}
+
+function checkPermissionsPolicy(value: string | undefined): HeaderStatus {
+  if (!value) return { present: false, value: null, valid: false, issues: ['Missing Permissions-Policy'] };
+  return { present: true, value, valid: true, issues: [] };
+}

package/src/audit/index.ts

@@ -0,0 +1,66 @@
+import { resolveDns } from './dns.js';
+import { analyzeTransport } from './transport.js';
+import { analyzeHeaders } from './headers.js';
+import { calculateScore } from './scoring.js';
+import { AuditResult, AuditOptions } from './types.js';
+import { URL } from 'node:url';
+import { IPGuard } from '../core/security/ipGuard.js';
+
+export async function auditUrl(urlStr: string, options: AuditOptions = {}): Promise<AuditResult> {
+  const timeout = options.timeout || 10000;
+
+  // 1. Basic URL validation
+  let url: URL;
+  try {
+    url = new URL(urlStr);
+    if (!['http:', 'https:'].includes(url.protocol)) {
+      throw new Error('Only HTTP and HTTPS protocols are supported');
+    }
+  } catch (error: any) {
+    throw new Error(`Invalid URL: ${error.message}`, { cause: error });
+  }
+
+  // 2. SSRF Guard
+  const isSafe = await IPGuard.validateHost(url.hostname);
+  if (!isSafe) {
+    throw new Error('Access to internal or private infrastructure is prohibited');
+  }
+
+  // 3. Parallelize DNS and Transport
+  // We handle transport errors differently as they are fatal for the audit (e.g. connection refused)
+  // DNS errors might return partial results but usually if transport works, DNS worked (unless transport used IP)
+
+  const dnsPromise = resolveDns(url.hostname);
+  const transportPromise = analyzeTransport(urlStr, timeout);
+
+  const [dnsResult, transportResult] = await Promise.all([
+    dnsPromise,
+    transportPromise
+  ]);
+
+  // 3. Analyze Headers
+  const headersResult = analyzeHeaders(transportResult.transport.headers);
+
+  // 4. Calculate Score
+  const scoringResult = calculateScore(
+    transportResult.transport,
+    dnsResult,
+    headersResult,
+    transportResult.performance,
+    transportResult.issues
+  );
+
+  // 5. Build Result
+  const result: AuditResult = {
+    url: urlStr,
+    transport: transportResult.transport,
+    securityHeaders: headersResult,
+    dns: dnsResult,
+    performance: transportResult.performance,
+    score: scoringResult.score,
+    grade: scoringResult.grade,
+    issues: scoringResult.issues
+  };
+
+  return result;
+}

package/src/audit/scoring.ts

@@ -0,0 +1,232 @@
+/* eslint-disable no-useless-assignment */
+import { TransportDiagnostics, DnsDiagnostics, SecurityHeadersResult, PerformanceMetrics, AuditIssue } from './types.js';
+
+interface CategoryScores {
+  transport: number;
+  security: number;
+  performance: number;
+  infrastructure: number;
+}
+
+export function calculateScore(
+  transport: TransportDiagnostics,
+  dns: DnsDiagnostics,
+  headers: SecurityHeadersResult,
+  performance: PerformanceMetrics,
+  existingIssues: AuditIssue[]
+): { score: number; grade: 'A' | 'B' | 'C' | 'D' | 'F'; issues: AuditIssue[]; categoryScores: CategoryScores } {
+
+  const issues: AuditIssue[] = [...existingIssues];
+  let transportScore = 0; // Max 30
+  let securityScore = 0;  // Max 20
+  let performanceScore = 0; // Max 30
+  let infrastructureScore = 0; // Max 20
+
+  // 1. Transport Security (30 pts)
+  // TLS Version
+  if (transport.tlsVersion) {
+    const version = parseFloat(transport.tlsVersion.replace('v', '').replace('TLS', '').trim());
+    if (version >= 1.2) {
+      transportScore += 15;
+    } else {
+      issues.push({
+        id: 'tls-old',
+        severity: 'severe',
+        category: 'tls',
+        message: `Deprecated TLS version: ${transport.tlsVersion}`,
+        scorePenalty: 15
+      });
+    }
+  } else if (transport.certificate) {
+    // HTTPS but no version detected? Unlikely.
+  } else {
+     // HTTP only?
+     issues.push({
+       id: 'no-https',
+       severity: 'critical',
+       category: 'tls',
+       message: 'Site is not using HTTPS',
+       scorePenalty: 30
+     });
+  }
+
+  // Certificate
+  if (transport.certificate) {
+    if (transport.certificate.isValidChain && !transport.certificate.isSelfSigned) {
+      transportScore += 15;
+    } else {
+      // Already caught in transport.ts, but let's ensure score reflects it
+      // If issues has cert-invalid, we don't add points.
+    }
+
+    if (transport.certificate.daysUntilExpiry < 30 && transport.certificate.daysUntilExpiry >= 0) {
+       issues.push({
+         id: 'cert-expiring-soon',
+         severity: 'moderate',
+         category: 'tls',
+         message: `Certificate expires in ${transport.certificate.daysUntilExpiry} days`,
+         scorePenalty: 5
+       });
+       // Penalty applied to transport score logic implicitly by not reaching max,
+       // but here we are adding up points.
+       // Let's deduct from the 15 points we might have given.
+       transportScore -= 5;
+    } else if (transport.certificate.daysUntilExpiry < 0) {
+       issues.push({
+         id: 'cert-expired',
+         severity: 'critical',
+         category: 'tls',
+         message: `Certificate expired on ${transport.certificate.validTo}`,
+         scorePenalty: 30
+       });
+       transportScore = 0; // Reset transport score
+    }
+  }
+
+  // 2. Response Security (Headers) (20 pts)
+  // headers.score is 0-100. Map to 0-20.
+  securityScore = (headers.score / 100) * 20;
+
+  // Add issues for missing critical headers
+  if (!headers.strictTransportSecurity.present) {
+    issues.push({
+      id: 'hsts-missing',
+      severity: 'moderate',
+      category: 'headers',
+      message: 'Missing Strict-Transport-Security header',
+      scorePenalty: 5
+    });
+  }
+  if (!headers.contentSecurityPolicy.present) {
+    issues.push({
+        id: 'csp-missing',
+        severity: 'moderate',
+        category: 'headers',
+        message: 'Missing Content-Security-Policy header',
+        scorePenalty: 5
+      });
+  }
+
+  // 3. Performance (30 pts)
+  // HTTP/2 (5 pts)
+  if (transport.alpnProtocol === 'h2' || transport.httpVersion === '2.0') {
+    performanceScore += 5;
+  } else {
+    issues.push({
+      id: 'no-h2',
+      severity: 'minor',
+      category: 'performance',
+      message: 'HTTP/2 not supported',
+      scorePenalty: 5
+    });
+  }
+
+  // Compression (5 pts)
+  if (transport.compression.length > 0) {
+    performanceScore += 5;
+  } else {
+    issues.push({
+      id: 'no-compression',
+      severity: 'moderate',
+      category: 'performance',
+      message: 'No compression enabled (gzip/br)',
+      scorePenalty: 5
+    });
+  }
+
+  // TTFB (10 pts)
+  if (performance.ttfb < 800) {
+    performanceScore += 10;
+  } else {
+    issues.push({
+      id: 'slow-ttfb',
+      severity: 'moderate',
+      category: 'performance',
+      message: `Slow TTFB: ${performance.ttfb.toFixed(0)}ms`,
+      scorePenalty: 10
+    });
+  }
+
+  // Redirects (5 pts)
+  if (transport.redirectCount <= 3) {
+    performanceScore += 5;
+  } else {
+    issues.push({
+      id: 'too-many-redirects',
+      severity: 'moderate',
+      category: 'performance',
+      message: `Too many redirects: ${transport.redirectCount}`,
+      scorePenalty: 5
+    });
+  }
+
+  // HTML Size (5 pts)
+  if (performance.htmlSize < 1024 * 1024) { // 1MB
+    performanceScore += 5;
+  } else {
+     issues.push({
+      id: 'large-html',
+      severity: 'minor',
+      category: 'performance',
+      message: `HTML size > 1MB: ${(performance.htmlSize / 1024 / 1024).toFixed(2)}MB`,
+      scorePenalty: 5
+    });
+  }
+
+  // 4. Infrastructure (20 pts)
+  // IPv6 (10 pts)
+  if (dns.ipv6Support) {
+    infrastructureScore += 10;
+  } else {
+    issues.push({
+      id: 'no-ipv6',
+      severity: 'minor',
+      category: 'dns',
+      message: 'No IPv6 DNS records found',
+      scorePenalty: 5
+    });
+  }
+
+  // Redundancy (10 pts)
+  if (dns.ipCount > 1) {
+    infrastructureScore += 10;
+  } else {
+    issues.push({
+      id: 'single-ip',
+      severity: 'minor',
+      category: 'dns',
+      message: 'Single IP address detected (no redundancy)',
+      scorePenalty: 5
+    });
+  }
+
+  let totalScore = transportScore + securityScore + performanceScore + infrastructureScore;
+
+  // Critical Overrides
+  const criticalIssues = issues.filter(i => i.severity === 'critical');
+  if (criticalIssues.length > 0) {
+    totalScore = Math.min(totalScore, 39); // Cap at F (<40)
+  }
+
+  const grade = getGrade(totalScore);
+
+  return {
+    score: Math.round(totalScore),
+    grade,
+    issues,
+    categoryScores: {
+        transport: transportScore,
+        security: securityScore,
+        performance: performanceScore,
+        infrastructure: infrastructureScore
+    }
+  };
+}
+
+function getGrade(score: number): 'A' | 'B' | 'C' | 'D' | 'F' {
+  if (score >= 90) return 'A';
+  if (score >= 75) return 'B';
+  if (score >= 60) return 'C';
+  if (score >= 40) return 'D';
+  return 'F';
+}