@crawlith/core 0.1.1 → 0.1.2

package/src/analysis/scoring.ts

@@ -1,66 +0,0 @@
-import { Metrics } from '../graph/metrics.js';
-import type { PageAnalysis } from './analyze.js';
-
-export interface SiteScore {
-  seoHealthScore: number;
-  authorityEntropyOrphanScore: number;
-  overallScore: number;
-}
-
-export function scorePageSeo(page: PageAnalysis): number {
-  if (page.meta.crawlStatus === 'blocked_by_robots') {
-    return 0;
-  }
-  const titleMeta = (scoreTextStatus(page.title.status) + scoreTextStatus(page.metaDescription.status)) / 2;
-  const h1 = page.h1.status === 'ok' ? 100 : page.h1.status === 'warning' ? 60 : 10;
-  const wordQuality = Math.min(100, (page.content.wordCount / 600) * 100) * 0.7 + Math.min(100, page.content.textHtmlRatio * 500) * 0.3;
-  const thin = 100 - page.thinScore;
-  const imageDen = Math.max(1, page.images.totalImages);
-  const imageAlt = Math.max(0, 100 - ((page.images.missingAlt + page.images.emptyAlt) / imageDen) * 100);
-  const structured = page.structuredData.present ? (page.structuredData.valid ? 100 : 40) : 30;
-  const linkBalance = Math.max(0, 100 - Math.abs(page.links.externalRatio - 0.3) * 200);
-
-  const score =
-    titleMeta * 0.15 +
-    h1 * 0.1 +
-    wordQuality * 0.2 +
-    thin * 0.2 +
-    imageAlt * 0.1 +
-    structured * 0.1 +
-    linkBalance * 0.15;
-
-  return Number(Math.max(0, Math.min(100, score)).toFixed(2));
-}
-
-function scoreTextStatus(status: PageAnalysis['title']['status']): number {
-  switch (status) {
-    case 'ok': return 100;
-    case 'duplicate': return 45;
-    case 'too_short': return 60;
-    case 'too_long': return 60;
-    case 'missing': return 0;
-  }
-}
-
-export function aggregateSiteScore(metrics: Metrics, pages: PageAnalysis[]): SiteScore {
-  const seoHealthScore = pages.length === 0 ? 0 : pages.reduce((acc, page) => acc + page.seoScore, 0) / pages.length;
-
-  const avgAuthority = metrics.topAuthorityPages.length === 0
-    ? 0
-    : metrics.topAuthorityPages.reduce((acc, item) => acc + item.authority, 0) / metrics.topAuthorityPages.length;
-  const entropyScore = Math.max(0, 100 - Math.abs(metrics.structuralEntropy - 2) * 25);
-  const orphanPenalty = metrics.totalPages === 0 ? 0 : (metrics.orphanPages.length / metrics.totalPages) * 100;
-  const authorityEntropyOrphanScore = Math.max(0, Math.min(100, (avgAuthority * 100 * 0.4) + (entropyScore * 0.35) + ((100 - orphanPenalty) * 0.25)));
-
-  let overallScore = Number((seoHealthScore * 0.7 + authorityEntropyOrphanScore * 0.3).toFixed(2));
-
-  if (pages.some(p => p.meta.crawlStatus === 'blocked_by_robots')) {
-    overallScore = 0;
-  }
-
-  return {
-    seoHealthScore: Number(seoHealthScore.toFixed(2)),
-    authorityEntropyOrphanScore: Number(authorityEntropyOrphanScore.toFixed(2)),
-    overallScore
-  };
-}

package/src/analysis/seo.ts

@@ -1,82 +0,0 @@
-import { load } from 'cheerio';
-
-export type SeoStatus = 'ok' | 'missing' | 'too_short' | 'too_long' | 'duplicate';
-
-export interface TextFieldAnalysis {
-  value: string | null;
-  length: number;
-  status: SeoStatus;
-}
-
-export interface H1Analysis {
-  count: number;
-  status: 'ok' | 'critical' | 'warning';
-  matchesTitle: boolean;
-}
-
-function normalizedText(value: string | null): string {
-  return (value ?? '').trim().toLowerCase();
-}
-
-export function analyzeTitle(html: string): TextFieldAnalysis {
-  const $ = load(html);
-  const title = $('title').first().text().trim();
-  if (!title) {
-    return { value: null, length: 0, status: 'missing' };
-  }
-
-  if (title.length < 50) return { value: title, length: title.length, status: 'too_short' };
-  if (title.length > 60) return { value: title, length: title.length, status: 'too_long' };
-  return { value: title, length: title.length, status: 'ok' };
-}
-
-export function analyzeMetaDescription(html: string): TextFieldAnalysis {
-  const $ = load(html);
-  const raw = $('meta[name="description"]').attr('content');
-  if (raw === undefined) {
-    return { value: null, length: 0, status: 'missing' };
-  }
-
-  const description = raw.trim();
-  if (!description) {
-    return { value: '', length: 0, status: 'missing' };
-  }
-
-  if (description.length < 140) return { value: description, length: description.length, status: 'too_short' };
-  if (description.length > 160) return { value: description, length: description.length, status: 'too_long' };
-  return { value: description, length: description.length, status: 'ok' };
-}
-
-export function applyDuplicateStatuses<T extends TextFieldAnalysis>(fields: T[]): T[] {
-  const counts = new Map<string, number>();
-  for (const field of fields) {
-    const key = normalizedText(field.value);
-    if (!key) continue;
-    counts.set(key, (counts.get(key) || 0) + 1);
-  }
-
-  return fields.map((field) => {
-    const key = normalizedText(field.value);
-    if (!key) return field;
-    if ((counts.get(key) || 0) > 1) {
-      return { ...field, status: 'duplicate' };
-    }
-    return field;
-  });
-}
-
-export function analyzeH1(html: string, titleValue: string | null): H1Analysis {
-  const $ = load(html);
-  const h1Values = $('h1').toArray().map((el) => $(el).text().trim()).filter(Boolean);
-  const count = h1Values.length;
-  const first = h1Values[0] || null;
-  const matchesTitle = Boolean(first && titleValue && normalizedText(first) === normalizedText(titleValue));
-
-  if (count === 0) {
-    return { count, status: 'critical', matchesTitle };
-  }
-  if (count > 1) {
-    return { count, status: 'warning', matchesTitle };
-  }
-  return { count, status: 'ok', matchesTitle };
-}

package/src/analysis/structuredData.ts

@@ -1,62 +0,0 @@
-import { load } from 'cheerio';
-
-export interface StructuredDataResult {
-  present: boolean;
-  types: string[];
-  valid: boolean;
-}
-
-export function analyzeStructuredData(html: string): StructuredDataResult {
-  const $ = load(html);
-  const scripts = $('script[type="application/ld+json"]').toArray();
-  if (scripts.length === 0) {
-    return { present: false, types: [], valid: false };
-  }
-
-  const types = new Set<string>();
-  let valid = true;
-
-  for (const script of scripts) {
-    const raw = $(script).text().trim();
-    if (!raw) {
-      valid = false;
-      continue;
-    }
-
-    try {
-      const parsed = JSON.parse(raw);
-      extractTypes(parsed, types);
-    } catch {
-      valid = false;
-    }
-  }
-
-  return {
-    present: true,
-    valid,
-    types: Array.from(types)
-  };
-}
-
-function extractTypes(input: unknown, types: Set<string>): void {
-  if (Array.isArray(input)) {
-    input.forEach((item) => extractTypes(item, types));
-    return;
-  }
-
-  if (!input || typeof input !== 'object') return;
-
-  const maybeType = (input as Record<string, unknown>)['@type'];
-  if (typeof maybeType === 'string') {
-    types.add(maybeType);
-  } else if (Array.isArray(maybeType)) {
-    for (const item of maybeType) {
-      if (typeof item === 'string') types.add(item);
-    }
-  }
-
-  const graph = (input as Record<string, unknown>)['@graph'];
-  if (Array.isArray(graph)) {
-    graph.forEach((item) => extractTypes(item, types));
-  }
-}

package/src/analysis/templates.ts

@@ -1,9 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-export const ANALYSIS_LIST_TEMPLATE = fs.readFileSync(path.join(__dirname, 'analysis_list.html'), 'utf-8');
-export const ANALYSIS_PAGE_TEMPLATE = fs.readFileSync(path.join(__dirname, 'analysis_page.html'), 'utf-8');

package/src/audit/dns.ts

@@ -1,49 +0,0 @@
-import dns from 'node:dns/promises';
-import { DnsDiagnostics } from './types.js';
-
-export async function resolveDns(hostname: string): Promise<DnsDiagnostics> {
-  const start = performance.now();
-
-  const result: DnsDiagnostics = {
-    a: [],
-    aaaa: [],
-    cname: [],
-    reverse: [],
-    ipCount: 0,
-    ipv6Support: false,
-    resolutionTime: 0
-  };
-
-  try {
-    // We run these in parallel
-    const [a, aaaa, cname] = await Promise.all([
-      dns.resolve4(hostname).catch(() => [] as string[]),
-      dns.resolve6(hostname).catch(() => [] as string[]),
-      dns.resolveCname(hostname).catch(() => [] as string[])
-    ]);
-
-    result.a = a;
-    result.aaaa = aaaa;
-    result.cname = cname;
-    result.ipCount = a.length + aaaa.length;
-    result.ipv6Support = aaaa.length > 0;
-
-    // Try reverse lookup on first IP if available
-    const ipToReverse = a.length > 0 ? a[0] : (aaaa.length > 0 ? aaaa[0] : null);
-
-    if (ipToReverse) {
-      try {
-        result.reverse = await dns.reverse(ipToReverse);
-      } catch {
-        // Reverse lookup failed, ignore
-      }
-    }
-
-  } catch (_error) {
-    // DNS resolution failed entirely or other error
-    // We return empty result but with time measured
-  }
-
-  result.resolutionTime = performance.now() - start;
-  return result;
-}

package/src/audit/headers.ts

@@ -1,98 +0,0 @@
-import { SecurityHeadersResult, HeaderStatus } from './types.js';
-
-export function analyzeHeaders(headers: Record<string, string | string[] | undefined>): SecurityHeadersResult {
-  const normalized: Record<string, string> = {};
-  for (const [key, value] of Object.entries(headers)) {
-    if (typeof value === 'string') {
-      normalized[key.toLowerCase()] = value;
-    } else if (Array.isArray(value)) {
-      normalized[key.toLowerCase()] = value.join(', ');
-    }
-  }
-
-  const result: SecurityHeadersResult = {
-    strictTransportSecurity: checkHSTS(normalized['strict-transport-security']),
-    contentSecurityPolicy: checkCSP(normalized['content-security-policy']),
-    xFrameOptions: checkXFrameOptions(normalized['x-frame-options']),
-    xContentTypeOptions: checkXContentTypeOptions(normalized['x-content-type-options']),
-    referrerPolicy: checkReferrerPolicy(normalized['referrer-policy']),
-    permissionsPolicy: checkPermissionsPolicy(normalized['permissions-policy']),
-    details: normalized,
-    score: 0
-  };
-
-  // Calculate internal score (0-100) based on presence and validity
-  let score = 0;
-  const weights = {
-    hsts: 30,
-    csp: 25,
-    xframe: 15,
-    xcontent: 15,
-    referrer: 10,
-    permissions: 5
-  };
-
-  if (result.strictTransportSecurity.valid) score += weights.hsts;
-  if (result.contentSecurityPolicy.valid) score += weights.csp;
-  if (result.xFrameOptions.valid) score += weights.xframe;
-  if (result.xContentTypeOptions.valid) score += weights.xcontent;
-  if (result.referrerPolicy.valid) score += weights.referrer;
-  if (result.permissionsPolicy.valid) score += weights.permissions;
-
-  result.score = score;
-
-  return result;
-}
-
-function checkHSTS(value: string | undefined): HeaderStatus {
-  if (!value) return { present: false, value: null, valid: false, issues: ['Missing HSTS header'] };
-
-  const valid = value.includes('max-age=');
-  const issues: string[] = [];
-  if (!valid) issues.push('Missing max-age directive');
-  if (!value.includes('includeSubDomains')) issues.push('Missing includeSubDomains');
-
-  return { present: true, value, valid, issues };
-}
-
-function checkCSP(value: string | undefined): HeaderStatus {
-  if (!value) return { present: false, value: null, valid: false, issues: ['Missing CSP header'] };
-
-  // Basic check: non-empty
-  return { present: true, value, valid: value.length > 0, issues: [] };
-}
-
-function checkXFrameOptions(value: string | undefined): HeaderStatus {
-  if (!value) return { present: false, value: null, valid: false, issues: ['Missing X-Frame-Options'] };
-
-  const upper = value.toUpperCase();
-  const valid = upper === 'DENY' || upper === 'SAMEORIGIN';
-  return {
-    present: true,
-    value,
-    valid,
-    issues: valid ? [] : [`Invalid value: ${value}`]
-  };
-}
-
-function checkXContentTypeOptions(value: string | undefined): HeaderStatus {
-  if (!value) return { present: false, value: null, valid: false, issues: ['Missing X-Content-Type-Options'] };
-
-  const valid = value.toLowerCase() === 'nosniff';
-  return {
-    present: true,
-    value,
-    valid,
-    issues: valid ? [] : [`Invalid value: ${value}`]
-  };
-}
-
-function checkReferrerPolicy(value: string | undefined): HeaderStatus {
-  if (!value) return { present: false, value: null, valid: false, issues: ['Missing Referrer-Policy'] };
-  return { present: true, value, valid: true, issues: [] };
-}
-
-function checkPermissionsPolicy(value: string | undefined): HeaderStatus {
-  if (!value) return { present: false, value: null, valid: false, issues: ['Missing Permissions-Policy'] };
-  return { present: true, value, valid: true, issues: [] };
-}

package/src/audit/index.ts

@@ -1,66 +0,0 @@
-import { resolveDns } from './dns.js';
-import { analyzeTransport } from './transport.js';
-import { analyzeHeaders } from './headers.js';
-import { calculateScore } from './scoring.js';
-import { AuditResult, AuditOptions } from './types.js';
-import { URL } from 'node:url';
-import { IPGuard } from '../core/security/ipGuard.js';
-
-export async function auditUrl(urlStr: string, options: AuditOptions = {}): Promise<AuditResult> {
-  const timeout = options.timeout || 10000;
-
-  // 1. Basic URL validation
-  let url: URL;
-  try {
-    url = new URL(urlStr);
-    if (!['http:', 'https:'].includes(url.protocol)) {
-      throw new Error('Only HTTP and HTTPS protocols are supported');
-    }
-  } catch (error: any) {
-    throw new Error(`Invalid URL: ${error.message}`, { cause: error });
-  }
-
-  // 2. SSRF Guard
-  const isSafe = await IPGuard.validateHost(url.hostname);
-  if (!isSafe) {
-    throw new Error('Access to internal or private infrastructure is prohibited');
-  }
-
-  // 3. Parallelize DNS and Transport
-  // We handle transport errors differently as they are fatal for the audit (e.g. connection refused)
-  // DNS errors might return partial results but usually if transport works, DNS worked (unless transport used IP)
-
-  const dnsPromise = resolveDns(url.hostname);
-  const transportPromise = analyzeTransport(urlStr, timeout);
-
-  const [dnsResult, transportResult] = await Promise.all([
-    dnsPromise,
-    transportPromise
-  ]);
-
-  // 3. Analyze Headers
-  const headersResult = analyzeHeaders(transportResult.transport.headers);
-
-  // 4. Calculate Score
-  const scoringResult = calculateScore(
-    transportResult.transport,
-    dnsResult,
-    headersResult,
-    transportResult.performance,
-    transportResult.issues
-  );
-
-  // 5. Build Result
-  const result: AuditResult = {
-    url: urlStr,
-    transport: transportResult.transport,
-    securityHeaders: headersResult,
-    dns: dnsResult,
-    performance: transportResult.performance,
-    score: scoringResult.score,
-    grade: scoringResult.grade,
-    issues: scoringResult.issues
-  };
-
-  return result;
-}

package/src/audit/scoring.ts

@@ -1,232 +0,0 @@
-/* eslint-disable no-useless-assignment */
-import { TransportDiagnostics, DnsDiagnostics, SecurityHeadersResult, PerformanceMetrics, AuditIssue } from './types.js';
-
-interface CategoryScores {
-  transport: number;
-  security: number;
-  performance: number;
-  infrastructure: number;
-}
-
-export function calculateScore(
-  transport: TransportDiagnostics,
-  dns: DnsDiagnostics,
-  headers: SecurityHeadersResult,
-  performance: PerformanceMetrics,
-  existingIssues: AuditIssue[]
-): { score: number; grade: 'A' | 'B' | 'C' | 'D' | 'F'; issues: AuditIssue[]; categoryScores: CategoryScores } {
-
-  const issues: AuditIssue[] = [...existingIssues];
-  let transportScore = 0; // Max 30
-  let securityScore = 0;  // Max 20
-  let performanceScore = 0; // Max 30
-  let infrastructureScore = 0; // Max 20
-
-  // 1. Transport Security (30 pts)
-  // TLS Version
-  if (transport.tlsVersion) {
-    const version = parseFloat(transport.tlsVersion.replace('v', '').replace('TLS', '').trim());
-    if (version >= 1.2) {
-      transportScore += 15;
-    } else {
-      issues.push({
-        id: 'tls-old',
-        severity: 'severe',
-        category: 'tls',
-        message: `Deprecated TLS version: ${transport.tlsVersion}`,
-        scorePenalty: 15
-      });
-    }
-  } else if (transport.certificate) {
-    // HTTPS but no version detected? Unlikely.
-  } else {
-     // HTTP only?
-     issues.push({
-       id: 'no-https',
-       severity: 'critical',
-       category: 'tls',
-       message: 'Site is not using HTTPS',
-       scorePenalty: 30
-     });
-  }
-
-  // Certificate
-  if (transport.certificate) {
-    if (transport.certificate.isValidChain && !transport.certificate.isSelfSigned) {
-      transportScore += 15;
-    } else {
-      // Already caught in transport.ts, but let's ensure score reflects it
-      // If issues has cert-invalid, we don't add points.
-    }
-
-    if (transport.certificate.daysUntilExpiry < 30 && transport.certificate.daysUntilExpiry >= 0) {
-       issues.push({
-         id: 'cert-expiring-soon',
-         severity: 'moderate',
-         category: 'tls',
-         message: `Certificate expires in ${transport.certificate.daysUntilExpiry} days`,
-         scorePenalty: 5
-       });
-       // Penalty applied to transport score logic implicitly by not reaching max,
-       // but here we are adding up points.
-       // Let's deduct from the 15 points we might have given.
-       transportScore -= 5;
-    } else if (transport.certificate.daysUntilExpiry < 0) {
-       issues.push({
-         id: 'cert-expired',
-         severity: 'critical',
-         category: 'tls',
-         message: `Certificate expired on ${transport.certificate.validTo}`,
-         scorePenalty: 30
-       });
-       transportScore = 0; // Reset transport score
-    }
-  }
-
-  // 2. Response Security (Headers) (20 pts)
-  // headers.score is 0-100. Map to 0-20.
-  securityScore = (headers.score / 100) * 20;
-
-  // Add issues for missing critical headers
-  if (!headers.strictTransportSecurity.present) {
-    issues.push({
-      id: 'hsts-missing',
-      severity: 'moderate',
-      category: 'headers',
-      message: 'Missing Strict-Transport-Security header',
-      scorePenalty: 5
-    });
-  }
-  if (!headers.contentSecurityPolicy.present) {
-    issues.push({
-        id: 'csp-missing',
-        severity: 'moderate',
-        category: 'headers',
-        message: 'Missing Content-Security-Policy header',
-        scorePenalty: 5
-      });
-  }
-
-  // 3. Performance (30 pts)
-  // HTTP/2 (5 pts)
-  if (transport.alpnProtocol === 'h2' || transport.httpVersion === '2.0') {
-    performanceScore += 5;
-  } else {
-    issues.push({
-      id: 'no-h2',
-      severity: 'minor',
-      category: 'performance',
-      message: 'HTTP/2 not supported',
-      scorePenalty: 5
-    });
-  }
-
-  // Compression (5 pts)
-  if (transport.compression.length > 0) {
-    performanceScore += 5;
-  } else {
-    issues.push({
-      id: 'no-compression',
-      severity: 'moderate',
-      category: 'performance',
-      message: 'No compression enabled (gzip/br)',
-      scorePenalty: 5
-    });
-  }
-
-  // TTFB (10 pts)
-  if (performance.ttfb < 800) {
-    performanceScore += 10;
-  } else {
-    issues.push({
-      id: 'slow-ttfb',
-      severity: 'moderate',
-      category: 'performance',
-      message: `Slow TTFB: ${performance.ttfb.toFixed(0)}ms`,
-      scorePenalty: 10
-    });
-  }
-
-  // Redirects (5 pts)
-  if (transport.redirectCount <= 3) {
-    performanceScore += 5;
-  } else {
-    issues.push({
-      id: 'too-many-redirects',
-      severity: 'moderate',
-      category: 'performance',
-      message: `Too many redirects: ${transport.redirectCount}`,
-      scorePenalty: 5
-    });
-  }
-
-  // HTML Size (5 pts)
-  if (performance.htmlSize < 1024 * 1024) { // 1MB
-    performanceScore += 5;
-  } else {
-     issues.push({
-      id: 'large-html',
-      severity: 'minor',
-      category: 'performance',
-      message: `HTML size > 1MB: ${(performance.htmlSize / 1024 / 1024).toFixed(2)}MB`,
-      scorePenalty: 5
-    });
-  }
-
-  // 4. Infrastructure (20 pts)
-  // IPv6 (10 pts)
-  if (dns.ipv6Support) {
-    infrastructureScore += 10;
-  } else {
-    issues.push({
-      id: 'no-ipv6',
-      severity: 'minor',
-      category: 'dns',
-      message: 'No IPv6 DNS records found',
-      scorePenalty: 5
-    });
-  }
-
-  // Redundancy (10 pts)
-  if (dns.ipCount > 1) {
-    infrastructureScore += 10;
-  } else {
-    issues.push({
-      id: 'single-ip',
-      severity: 'minor',
-      category: 'dns',
-      message: 'Single IP address detected (no redundancy)',
-      scorePenalty: 5
-    });
-  }
-
-  let totalScore = transportScore + securityScore + performanceScore + infrastructureScore;
-
-  // Critical Overrides
-  const criticalIssues = issues.filter(i => i.severity === 'critical');
-  if (criticalIssues.length > 0) {
-    totalScore = Math.min(totalScore, 39); // Cap at F (<40)
-  }
-
-  const grade = getGrade(totalScore);
-
-  return {
-    score: Math.round(totalScore),
-    grade,
-    issues,
-    categoryScores: {
-        transport: transportScore,
-        security: securityScore,
-        performance: performanceScore,
-        infrastructure: infrastructureScore
-    }
-  };
-}
-
-function getGrade(score: number): 'A' | 'B' | 'C' | 'D' | 'F' {
-  if (score >= 90) return 'A';
-  if (score >= 75) return 'B';
-  if (score >= 60) return 'C';
-  if (score >= 40) return 'D';
-  return 'F';
-}