@crawlith/core 0.1.0

package/tests/audit/scoring.test.ts

@@ -0,0 +1,133 @@
+import { describe, it, expect } from 'vitest';
+import { calculateScore } from '../../src/audit/scoring.js';
+import { TransportDiagnostics, DnsDiagnostics, SecurityHeadersResult, PerformanceMetrics, AuditIssue } from '../../src/audit/types.js';
+
+describe('Scoring Engine', () => {
+  const mockTransport: TransportDiagnostics = {
+    tlsVersion: 'TLSv1.3',
+    cipherSuite: 'TLS_AES_256_GCM_SHA384',
+    alpnProtocol: 'h2',
+    certificate: {
+      issuer: 'Let\'s Encrypt',
+      subject: 'example.com',
+      validFrom: '2023-01-01',
+      validTo: '2024-01-01',
+      daysUntilExpiry: 60,
+      isSelfSigned: false,
+      isValidChain: true,
+      fingerprint: 'SHA256:...'
+    } as any,
+    httpVersion: '2.0',
+    compression: ['gzip'],
+    keepAlive: true,
+    transferEncoding: null,
+    redirectCount: 0,
+    redirects: [],
+    serverHeader: 'nginx',
+    headers: {}
+  };
+
+  const mockDns: DnsDiagnostics = {
+    a: ['1.1.1.1', '1.0.0.1'],
+    aaaa: ['2606:4700:4700::1111'],
+    cname: [],
+    reverse: [],
+    ipCount: 3,
+    ipv6Support: true,
+    resolutionTime: 10
+  };
+
+  const mockHeaders: SecurityHeadersResult = {
+    strictTransportSecurity: { present: true, valid: true, value: 'max-age=31536000' },
+    contentSecurityPolicy: { present: true, valid: true, value: "default-src 'self'" },
+    xFrameOptions: { present: true, valid: true, value: 'DENY' },
+    xContentTypeOptions: { present: true, valid: true, value: 'nosniff' },
+    referrerPolicy: { present: true, valid: true, value: 'strict-origin' },
+    permissionsPolicy: { present: true, valid: true, value: 'geolocation=()' },
+    details: {},
+    score: 100
+  };
+
+  const mockPerformance: PerformanceMetrics = {
+    dnsLookupTime: 10,
+    tcpConnectTime: 20,
+    tlsHandshakeTime: 30,
+    ttfb: 100,
+    totalTime: 200,
+    htmlSize: 50000,
+    headerSize: 500,
+    redirectTime: 0
+  };
+
+  it('should give perfect score for perfect inputs', () => {
+    const result = calculateScore(mockTransport, mockDns, mockHeaders, mockPerformance, []);
+    expect(result.score).toBe(100);
+    expect(result.grade).toBe('A');
+    expect(result.issues).toHaveLength(0);
+  });
+
+  it('should penalize TLS < 1.2', () => {
+    const badTransport = { ...mockTransport, tlsVersion: 'TLSv1.1' };
+    const result = calculateScore(badTransport, mockDns, mockHeaders, mockPerformance, []);
+    expect(result.score).toBeLessThan(100);
+    expect(result.categoryScores.transport).toBeLessThan(30);
+    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'tls-old' })]));
+  });
+
+  it('should penalize missing HTTPS', () => {
+    const badTransport = { ...mockTransport, tlsVersion: null, certificate: null };
+    const result = calculateScore(badTransport, mockDns, mockHeaders, mockPerformance, []);
+    expect(result.score).toBeLessThan(50); // Critical
+    expect(result.grade).toBe('F');
+    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'no-https' })]));
+  });
+
+  it('should fail on expired cert', () => {
+    const expiredTransport = {
+        ...mockTransport,
+        certificate: { ...mockTransport.certificate!, daysUntilExpiry: -5, validTo: '2023-01-01' }
+    };
+    const result = calculateScore(expiredTransport, mockDns, mockHeaders, mockPerformance, []);
+    expect(result.grade).toBe('F');
+    expect(result.score).toBeLessThanOrEqual(40);
+    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'cert-expired' })]));
+  });
+
+  it('should penalize missing security headers', () => {
+    // If score is 50, it means we lost 50 points in headers category (internal score)
+    // headers category is 20 points total. So we lose 10 points.
+    const badHeaders = { ...mockHeaders, score: 50, strictTransportSecurity: { present: false, valid: false, value: null } };
+    const result = calculateScore(mockTransport, mockDns, badHeaders, mockPerformance, []);
+    expect(result.categoryScores.security).toBe(10);
+    expect(result.score).toBe(90); // 100 - 10
+    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'hsts-missing' })]));
+  });
+
+  it('should penalize poor performance', () => {
+      const badPerf = { ...mockPerformance, ttfb: 1000, htmlSize: 2000000 };
+      const result = calculateScore(mockTransport, mockDns, mockHeaders, badPerf, []);
+      // TTFB > 800: Lose 10 pts
+      // HTML > 1MB: Lose 5 pts
+      // Total perf score (30) -> 15.
+      expect(result.categoryScores.performance).toBe(15);
+      expect(result.score).toBe(85);
+      expect(result.issues).toEqual(expect.arrayContaining([
+          expect.objectContaining({ id: 'slow-ttfb' }),
+          expect.objectContaining({ id: 'large-html' })
+      ]));
+  });
+
+  it('should penalize infrastructure issues', () => {
+      const badDns = { ...mockDns, ipv6Support: false, ipCount: 1 };
+      const result = calculateScore(mockTransport, badDns, mockHeaders, mockPerformance, []);
+      // No IPv6: Lose 10 pts
+      // Single IP: Lose 10 pts
+      // Infra score (20) -> 0.
+      expect(result.categoryScores.infrastructure).toBe(0);
+      expect(result.score).toBe(80);
+      expect(result.issues).toEqual(expect.arrayContaining([
+          expect.objectContaining({ id: 'no-ipv6' }),
+          expect.objectContaining({ id: 'single-ip' })
+      ]));
+  });
+});

package/tests/audit/security.test.ts

@@ -0,0 +1,12 @@
+import { describe, it, expect } from 'vitest';
+import { auditUrl } from '../../src/audit/index.js';
+
+describe('Audit Security', () => {
+    it('should block audits of internal IP addresses', async () => {
+        await expect(auditUrl('http://127.0.0.1')).rejects.toThrow('Access to internal or private infrastructure is prohibited');
+    });
+
+    it('should block audits of link-local addresses', async () => {
+        await expect(auditUrl('http://169.254.169.254')).rejects.toThrow('Access to internal or private infrastructure is prohibited');
+    });
+});

package/tests/audit/transport.test.ts

@@ -0,0 +1,112 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { analyzeTransport } from '../../src/audit/transport.js';
+import https from 'node:https';
+import http from 'node:http';
+import tls from 'node:tls';
+import { EventEmitter } from 'events';
+
+vi.mock('node:https');
+vi.mock('node:http');
+
+describe('Transport Diagnostics', () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should analyze HTTPS transport', async () => {
+    // Mock Response
+    const mockRes = new EventEmitter() as any;
+    mockRes.statusCode = 200;
+    mockRes.statusMessage = 'OK';
+    mockRes.headers = {
+      'content-encoding': 'gzip',
+      'server': 'nginx',
+      'connection': 'keep-alive'
+    };
+    mockRes.httpVersion = '1.1';
+
+    const mockSocket = new EventEmitter();
+    Object.setPrototypeOf(mockSocket, tls.TLSSocket.prototype);
+    (mockSocket as any).getPeerCertificate = () => ({
+      subject: { CN: 'example.com' },
+      issuer: { CN: 'Let\'s Encrypt' },
+      valid_from: 'Jan 1 2023',
+      valid_to: 'Jan 1 2024',
+      fingerprint: 'SHA256:...'
+    });
+    (mockSocket as any).getProtocol = () => 'TLSv1.3';
+    (mockSocket as any).getCipher = () => ({ name: 'TLS_AES_...' });
+    (mockSocket as any).alpnProtocol = 'h2';
+    (mockSocket as any).authorized = true;
+
+    mockRes.socket = mockSocket;
+
+    // Mock Request
+    const mockReq = new EventEmitter() as any;
+    mockReq.end = vi.fn();
+    mockReq.destroy = vi.fn();
+
+    // Mock https.request
+    vi.spyOn(https, 'request').mockImplementation((url, options, cb) => {
+      if (cb) cb(mockRes);
+      // Simulate socket events
+      setTimeout(() => {
+        mockReq.emit('socket', mockRes.socket);
+        mockRes.socket.emit('lookup');
+        mockRes.socket.emit('connect');
+        mockRes.socket.emit('secureConnect');
+        mockReq.emit('finish');
+        // Response data
+        mockRes.emit('data', Buffer.from('<html></html>'));
+        mockRes.emit('end');
+      }, 10);
+      return mockReq;
+    });
+
+    const result = await analyzeTransport('https://example.com', 1000);
+    expect(result.transport.tlsVersion).toBe('TLSv1.3');
+    expect(result.transport.httpVersion).toBe('1.1');
+    expect(result.performance.htmlSize).toBeGreaterThan(0);
+    expect(result.transport.headers['server']).toBe('nginx');
+  });
+
+  it('should handle redirects', async () => {
+    const req1 = new EventEmitter() as any; req1.end = vi.fn(); req1.destroy = vi.fn();
+    const res1 = new EventEmitter() as any; res1.statusCode = 301; res1.headers = { location: 'https://example.com/' };
+    res1.socket = new EventEmitter(); Object.setPrototypeOf(res1.socket, tls.TLSSocket.prototype);
+
+    const req2 = new EventEmitter() as any; req2.end = vi.fn(); req2.destroy = vi.fn();
+    const res2 = new EventEmitter() as any; res2.statusCode = 200; res2.headers = {};
+    res2.socket = new EventEmitter(); Object.setPrototypeOf(res2.socket, tls.TLSSocket.prototype);
+
+    // Setup res2 socket for TLS checks
+    res2.socket.getPeerCertificate = () => ({});
+    res2.socket.getProtocol = () => 'TLSv1.2';
+    res2.socket.getCipher = () => ({ name: 'AES' });
+
+    const requestSpy = vi.spyOn(https, 'request');
+    requestSpy
+      .mockImplementationOnce((url, options, cb) => {
+        if (cb) cb(res1);
+        setTimeout(() => {
+           req1.emit('socket', res1.socket);
+           res1.emit('data', Buffer.from('redirecting'));
+           res1.emit('end');
+        }, 10);
+        return req1;
+      })
+      .mockImplementationOnce((url, options, cb) => {
+        if (cb) cb(res2);
+        setTimeout(() => {
+           req2.emit('socket', res2.socket);
+           res2.emit('data', Buffer.from('ok'));
+           res2.emit('end');
+        }, 10);
+        return req2;
+      });
+
+    const result = await analyzeTransport('https://redirect.com', 1000);
+    expect(result.transport.redirectCount).toBe(1);
+    expect(result.transport.redirects[0].location).toBe('https://example.com/');
+  });
+});

package/tests/clustering.test.ts

@@ -0,0 +1,118 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { Graph } from '../src/graph/graph.js';
+import { detectContentClusters } from '../src/graph/cluster.js';
+
+describe('Content Clustering', () => {
+    let graph: Graph;
+
+    beforeEach(() => {
+        graph = new Graph();
+    });
+
+    it('should group similar pages into a cluster', () => {
+        // Mock simhashes for similar pages (Hamming distance 1)
+        const h1 = 0b101010n;
+        const h2 = 0b101011n;
+        const h3 = 0b101001n;
+
+        graph.addNode('https://example.com/p1', 0, 200);
+        graph.addNode('https://example.com/p2', 0, 200);
+        graph.addNode('https://example.com/p3', 0, 200);
+
+        graph.updateNodeData('https://example.com/p1', { simhash: h1.toString() });
+        graph.updateNodeData('https://example.com/p2', { simhash: h2.toString() });
+        graph.updateNodeData('https://example.com/p3', { simhash: h3.toString() });
+
+        const clusters = detectContentClusters(graph, 2, 2);
+
+        expect(clusters.length).toBe(1);
+        expect(clusters[0].count).toBe(3);
+        expect(graph.nodes.get('https://example.com/p1')?.clusterId).toBe(1);
+    });
+
+    it('should separate dissimilar pages', () => {
+        // Mock simhashes for very different pages
+        const h1 = 0b1111111111n;
+        const h2 = 0b0000000000n;
+
+        graph.addNode('https://example.com/p1', 0, 200);
+        graph.addNode('https://example.com/p2', 0, 200);
+
+        graph.updateNodeData('https://example.com/p1', { simhash: h1.toString() });
+        graph.updateNodeData('https://example.com/p2', { simhash: h2.toString() });
+
+        const clusters = detectContentClusters(graph, 2, 2);
+
+        expect(clusters.length).toBe(0); // None meet minSize 2
+    });
+
+    it('should respect minClusterSize', () => {
+        const h1 = 0b1n;
+        const h2 = 0b0n;
+
+        graph.addNode('https://example.com/p1', 0, 200);
+        graph.addNode('https://example.com/p2', 0, 200);
+
+        graph.updateNodeData('https://example.com/p1', { simhash: h1.toString() });
+        graph.updateNodeData('https://example.com/p2', { simhash: h2.toString() });
+
+        const clusters = detectContentClusters(graph, 1, 3);
+        expect(clusters.length).toBe(0);
+    });
+
+    it('should identify shared path prefixes (silos)', () => {
+        graph.addNode('https://example.com/blog/seo-tips', 0, 200);
+        graph.addNode('https://example.com/blog/link-building', 0, 200);
+        graph.addNode('https://example.com/blog/technical-seo', 0, 200);
+
+        const h = 0b111n;
+        graph.updateNodeData('https://example.com/blog/seo-tips', { simhash: h.toString() });
+        graph.updateNodeData('https://example.com/blog/link-building', { simhash: h.toString() });
+        graph.updateNodeData('https://example.com/blog/technical-seo', { simhash: h.toString() });
+
+        const clusters = detectContentClusters(graph, 0, 3);
+        expect(clusters[0].sharedPathPrefix).toBe('/blog');
+    });
+
+    it('should be deterministic with unstable input order', () => {
+        // We'll add nodes in different orders and check if cluster primary is same
+        const h = 0b111n;
+        graph.addNode('https://example.com/z', 0, 200);
+        graph.addNode('https://example.com/a', 0, 200);
+        graph.addNode('https://example.com/m', 0, 200);
+
+        graph.updateNodeData('https://example.com/z', { simhash: h.toString(), pageRank: 10 });
+        graph.updateNodeData('https://example.com/a', { simhash: h.toString(), pageRank: 10 });
+        graph.updateNodeData('https://example.com/m', { simhash: h.toString(), pageRank: 10 });
+
+        const clusters = detectContentClusters(graph, 0, 3);
+        // a should be primary because it's shortest/lexicographic first since PageRanks are same
+        expect(clusters[0].primaryUrl).toBe('https://example.com/a');
+    });
+
+    it('should use band optimization correctly (heuristic nature)', () => {
+        // Create many nodes in 2 groups
+        // Group 1: Matches in band 0
+        // Group 2: Matches in band 1
+        for (let i = 0; i < 5; i++) {
+            const url = `https://example.com/g1/${i}`;
+            graph.addNode(url, 0, 200);
+            // Simhash that matches in first 16 bits (0xAAAA)
+            const hash = BigInt(0xAAAA) | (BigInt(i) << 16n);
+            graph.updateNodeData(url, { simhash: hash.toString() });
+        }
+
+        for (let i = 0; i < 5; i++) {
+            const url = `https://example.com/g2/${i}`;
+            graph.addNode(url, 0, 200);
+            // Simhash that matches in second 16 bits (0xBBBB << 16)
+            const hash = (BigInt(0xBBBB) << 16n) | BigInt(i);
+            graph.updateNodeData(url, { simhash: hash.toString() });
+        }
+
+        const clusters = detectContentClusters(graph, 5, 3);
+        expect(clusters.length).toBe(2);
+        expect(clusters[0].count).toBe(5);
+        expect(clusters[1].count).toBe(5);
+    });
+});