@crawlith/core 0.1.1 → 0.1.2

package/tests/audit/scoring.test.ts

@@ -1,133 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { calculateScore } from '../../src/audit/scoring.js';
-import { TransportDiagnostics, DnsDiagnostics, SecurityHeadersResult, PerformanceMetrics } from '../../src/audit/types.js';
-
-describe('Scoring Engine', () => {
-  const mockTransport: TransportDiagnostics = {
-    tlsVersion: 'TLSv1.3',
-    cipherSuite: 'TLS_AES_256_GCM_SHA384',
-    alpnProtocol: 'h2',
-    certificate: {
-      issuer: 'Let\'s Encrypt',
-      subject: 'example.com',
-      validFrom: '2023-01-01',
-      validTo: '2024-01-01',
-      daysUntilExpiry: 60,
-      isSelfSigned: false,
-      isValidChain: true,
-      fingerprint: 'SHA256:...'
-    } as any,
-    httpVersion: '2.0',
-    compression: ['gzip'],
-    keepAlive: true,
-    transferEncoding: null,
-    redirectCount: 0,
-    redirects: [],
-    serverHeader: 'nginx',
-    headers: {}
-  };
-
-  const mockDns: DnsDiagnostics = {
-    a: ['1.1.1.1', '1.0.0.1'],
-    aaaa: ['2606:4700:4700::1111'],
-    cname: [],
-    reverse: [],
-    ipCount: 3,
-    ipv6Support: true,
-    resolutionTime: 10
-  };
-
-  const mockHeaders: SecurityHeadersResult = {
-    strictTransportSecurity: { present: true, valid: true, value: 'max-age=31536000' },
-    contentSecurityPolicy: { present: true, valid: true, value: "default-src 'self'" },
-    xFrameOptions: { present: true, valid: true, value: 'DENY' },
-    xContentTypeOptions: { present: true, valid: true, value: 'nosniff' },
-    referrerPolicy: { present: true, valid: true, value: 'strict-origin' },
-    permissionsPolicy: { present: true, valid: true, value: 'geolocation=()' },
-    details: {},
-    score: 100
-  };
-
-  const mockPerformance: PerformanceMetrics = {
-    dnsLookupTime: 10,
-    tcpConnectTime: 20,
-    tlsHandshakeTime: 30,
-    ttfb: 100,
-    totalTime: 200,
-    htmlSize: 50000,
-    headerSize: 500,
-    redirectTime: 0
-  };
-
-  it('should give perfect score for perfect inputs', () => {
-    const result = calculateScore(mockTransport, mockDns, mockHeaders, mockPerformance, []);
-    expect(result.score).toBe(100);
-    expect(result.grade).toBe('A');
-    expect(result.issues).toHaveLength(0);
-  });
-
-  it('should penalize TLS < 1.2', () => {
-    const badTransport = { ...mockTransport, tlsVersion: 'TLSv1.1' };
-    const result = calculateScore(badTransport, mockDns, mockHeaders, mockPerformance, []);
-    expect(result.score).toBeLessThan(100);
-    expect(result.categoryScores.transport).toBeLessThan(30);
-    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'tls-old' })]));
-  });
-
-  it('should penalize missing HTTPS', () => {
-    const badTransport = { ...mockTransport, tlsVersion: null, certificate: null };
-    const result = calculateScore(badTransport, mockDns, mockHeaders, mockPerformance, []);
-    expect(result.score).toBeLessThan(50); // Critical
-    expect(result.grade).toBe('F');
-    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'no-https' })]));
-  });
-
-  it('should fail on expired cert', () => {
-    const expiredTransport = {
-      ...mockTransport,
-      certificate: { ...mockTransport.certificate!, daysUntilExpiry: -5, validTo: '2023-01-01' }
-    };
-    const result = calculateScore(expiredTransport, mockDns, mockHeaders, mockPerformance, []);
-    expect(result.grade).toBe('F');
-    expect(result.score).toBeLessThanOrEqual(40);
-    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'cert-expired' })]));
-  });
-
-  it('should penalize missing security headers', () => {
-    // If score is 50, it means we lost 50 points in headers category (internal score)
-    // headers category is 20 points total. So we lose 10 points.
-    const badHeaders = { ...mockHeaders, score: 50, strictTransportSecurity: { present: false, valid: false, value: null } };
-    const result = calculateScore(mockTransport, mockDns, badHeaders, mockPerformance, []);
-    expect(result.categoryScores.security).toBe(10);
-    expect(result.score).toBe(90); // 100 - 10
-    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'hsts-missing' })]));
-  });
-
-  it('should penalize poor performance', () => {
-    const badPerf = { ...mockPerformance, ttfb: 1000, htmlSize: 2000000 };
-    const result = calculateScore(mockTransport, mockDns, mockHeaders, badPerf, []);
-    // TTFB > 800: Lose 10 pts
-    // HTML > 1MB: Lose 5 pts
-    // Total perf score (30) -> 15.
-    expect(result.categoryScores.performance).toBe(15);
-    expect(result.score).toBe(85);
-    expect(result.issues).toEqual(expect.arrayContaining([
-      expect.objectContaining({ id: 'slow-ttfb' }),
-      expect.objectContaining({ id: 'large-html' })
-    ]));
-  });
-
-  it('should penalize infrastructure issues', () => {
-    const badDns = { ...mockDns, ipv6Support: false, ipCount: 1 };
-    const result = calculateScore(mockTransport, badDns, mockHeaders, mockPerformance, []);
-    // No IPv6: Lose 10 pts
-    // Single IP: Lose 10 pts
-    // Infra score (20) -> 0.
-    expect(result.categoryScores.infrastructure).toBe(0);
-    expect(result.score).toBe(80);
-    expect(result.issues).toEqual(expect.arrayContaining([
-      expect.objectContaining({ id: 'no-ipv6' }),
-      expect.objectContaining({ id: 'single-ip' })
-    ]));
-  });
-});

package/tests/audit/security.test.ts

@@ -1,12 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { auditUrl } from '../../src/audit/index.js';
-
-describe('Audit Security', () => {
-    it('should block audits of internal IP addresses', async () => {
-        await expect(auditUrl('http://127.0.0.1')).rejects.toThrow('Access to internal or private infrastructure is prohibited');
-    });
-
-    it('should block audits of link-local addresses', async () => {
-        await expect(auditUrl('http://169.254.169.254')).rejects.toThrow('Access to internal or private infrastructure is prohibited');
-    });
-});

package/tests/audit/transport.test.ts

@@ -1,111 +0,0 @@
-import { describe, it, expect, vi, afterEach } from 'vitest';
-import { analyzeTransport } from '../../src/audit/transport.js';
-import https from 'node:https';
-import tls from 'node:tls';
-import { EventEmitter } from 'events';
-
-vi.mock('node:https');
-vi.mock('node:http');
-
-describe('Transport Diagnostics', () => {
-  afterEach(() => {
-    vi.clearAllMocks();
-  });
-
-  it('should analyze HTTPS transport', async () => {
-    // Mock Response
-    const mockRes = new EventEmitter() as any;
-    mockRes.statusCode = 200;
-    mockRes.statusMessage = 'OK';
-    mockRes.headers = {
-      'content-encoding': 'gzip',
-      'server': 'nginx',
-      'connection': 'keep-alive'
-    };
-    mockRes.httpVersion = '1.1';
-
-    const mockSocket = new EventEmitter();
-    Object.setPrototypeOf(mockSocket, tls.TLSSocket.prototype);
-    (mockSocket as any).getPeerCertificate = () => ({
-      subject: { CN: 'example.com' },
-      issuer: { CN: 'Let\'s Encrypt' },
-      valid_from: 'Jan 1 2023',
-      valid_to: 'Jan 1 2024',
-      fingerprint: 'SHA256:...'
-    });
-    (mockSocket as any).getProtocol = () => 'TLSv1.3';
-    (mockSocket as any).getCipher = () => ({ name: 'TLS_AES_...' });
-    (mockSocket as any).alpnProtocol = 'h2';
-    (mockSocket as any).authorized = true;
-
-    mockRes.socket = mockSocket;
-
-    // Mock Request
-    const mockReq = new EventEmitter() as any;
-    mockReq.end = vi.fn();
-    mockReq.destroy = vi.fn();
-
-    // Mock https.request
-    vi.spyOn(https, 'request').mockImplementation((url, options, cb) => {
-      if (cb) cb(mockRes);
-      // Simulate socket events
-      setTimeout(() => {
-        mockReq.emit('socket', mockRes.socket);
-        mockRes.socket.emit('lookup');
-        mockRes.socket.emit('connect');
-        mockRes.socket.emit('secureConnect');
-        mockReq.emit('finish');
-        // Response data
-        mockRes.emit('data', Buffer.from('<html></html>'));
-        mockRes.emit('end');
-      }, 10);
-      return mockReq;
-    });
-
-    const result = await analyzeTransport('https://example.com', 1000);
-    expect(result.transport.tlsVersion).toBe('TLSv1.3');
-    expect(result.transport.httpVersion).toBe('1.1');
-    expect(result.performance.htmlSize).toBeGreaterThan(0);
-    expect(result.transport.headers['server']).toBe('nginx');
-  });
-
-  it('should handle redirects', async () => {
-    const req1 = new EventEmitter() as any; req1.end = vi.fn(); req1.destroy = vi.fn();
-    const res1 = new EventEmitter() as any; res1.statusCode = 301; res1.headers = { location: 'https://example.com/' };
-    res1.socket = new EventEmitter(); Object.setPrototypeOf(res1.socket, tls.TLSSocket.prototype);
-
-    const req2 = new EventEmitter() as any; req2.end = vi.fn(); req2.destroy = vi.fn();
-    const res2 = new EventEmitter() as any; res2.statusCode = 200; res2.headers = {};
-    res2.socket = new EventEmitter(); Object.setPrototypeOf(res2.socket, tls.TLSSocket.prototype);
-
-    // Setup res2 socket for TLS checks
-    res2.socket.getPeerCertificate = () => ({});
-    res2.socket.getProtocol = () => 'TLSv1.2';
-    res2.socket.getCipher = () => ({ name: 'AES' });
-
-    const requestSpy = vi.spyOn(https, 'request');
-    requestSpy
-      .mockImplementationOnce((url, options, cb) => {
-        if (cb) cb(res1);
-        setTimeout(() => {
-           req1.emit('socket', res1.socket);
-           res1.emit('data', Buffer.from('redirecting'));
-           res1.emit('end');
-        }, 10);
-        return req1;
-      })
-      .mockImplementationOnce((url, options, cb) => {
-        if (cb) cb(res2);
-        setTimeout(() => {
-           req2.emit('socket', res2.socket);
-           res2.emit('data', Buffer.from('ok'));
-           res2.emit('end');
-        }, 10);
-        return req2;
-      });
-
-    const result = await analyzeTransport('https://redirect.com', 1000);
-    expect(result.transport.redirectCount).toBe(1);
-    expect(result.transport.redirects[0].location).toBe('https://example.com/');
-  });
-});

package/tests/clustering.test.ts

@@ -1,118 +0,0 @@
-import { describe, it, expect, beforeEach } from 'vitest';
-import { Graph } from '../src/graph/graph.js';
-import { detectContentClusters } from '../src/graph/cluster.js';
-
-describe('Content Clustering', () => {
-    let graph: Graph;
-
-    beforeEach(() => {
-        graph = new Graph();
-    });
-
-    it('should group similar pages into a cluster', () => {
-        // Mock simhashes for similar pages (Hamming distance 1)
-        const h1 = 0b101010n;
-        const h2 = 0b101011n;
-        const h3 = 0b101001n;
-
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-        graph.addNode('https://example.com/p3', 0, 200);
-
-        graph.updateNodeData('https://example.com/p1', { simhash: h1.toString() });
-        graph.updateNodeData('https://example.com/p2', { simhash: h2.toString() });
-        graph.updateNodeData('https://example.com/p3', { simhash: h3.toString() });
-
-        const clusters = detectContentClusters(graph, 2, 2);
-
-        expect(clusters.length).toBe(1);
-        expect(clusters[0].count).toBe(3);
-        expect(graph.nodes.get('https://example.com/p1')?.clusterId).toBe(1);
-    });
-
-    it('should separate dissimilar pages', () => {
-        // Mock simhashes for very different pages
-        const h1 = 0b1111111111n;
-        const h2 = 0b0000000000n;
-
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-
-        graph.updateNodeData('https://example.com/p1', { simhash: h1.toString() });
-        graph.updateNodeData('https://example.com/p2', { simhash: h2.toString() });
-
-        const clusters = detectContentClusters(graph, 2, 2);
-
-        expect(clusters.length).toBe(0); // None meet minSize 2
-    });
-
-    it('should respect minClusterSize', () => {
-        const h1 = 0b1n;
-        const h2 = 0b0n;
-
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-
-        graph.updateNodeData('https://example.com/p1', { simhash: h1.toString() });
-        graph.updateNodeData('https://example.com/p2', { simhash: h2.toString() });
-
-        const clusters = detectContentClusters(graph, 1, 3);
-        expect(clusters.length).toBe(0);
-    });
-
-    it('should identify shared path prefixes (silos)', () => {
-        graph.addNode('https://example.com/blog/seo-tips', 0, 200);
-        graph.addNode('https://example.com/blog/link-building', 0, 200);
-        graph.addNode('https://example.com/blog/technical-seo', 0, 200);
-
-        const h = 0b111n;
-        graph.updateNodeData('https://example.com/blog/seo-tips', { simhash: h.toString() });
-        graph.updateNodeData('https://example.com/blog/link-building', { simhash: h.toString() });
-        graph.updateNodeData('https://example.com/blog/technical-seo', { simhash: h.toString() });
-
-        const clusters = detectContentClusters(graph, 0, 3);
-        expect(clusters[0].sharedPathPrefix).toBe('/blog');
-    });
-
-    it('should be deterministic with unstable input order', () => {
-        // We'll add nodes in different orders and check if cluster primary is same
-        const h = 0b111n;
-        graph.addNode('https://example.com/z', 0, 200);
-        graph.addNode('https://example.com/a', 0, 200);
-        graph.addNode('https://example.com/m', 0, 200);
-
-        graph.updateNodeData('https://example.com/z', { simhash: h.toString(), pageRank: 10 });
-        graph.updateNodeData('https://example.com/a', { simhash: h.toString(), pageRank: 10 });
-        graph.updateNodeData('https://example.com/m', { simhash: h.toString(), pageRank: 10 });
-
-        const clusters = detectContentClusters(graph, 0, 3);
-        // a should be primary because it's shortest/lexicographic first since PageRanks are same
-        expect(clusters[0].primaryUrl).toBe('https://example.com/a');
-    });
-
-    it('should use band optimization correctly (heuristic nature)', () => {
-        // Create many nodes in 2 groups
-        // Group 1: Matches in band 0
-        // Group 2: Matches in band 1
-        for (let i = 0; i < 5; i++) {
-            const url = `https://example.com/g1/${i}`;
-            graph.addNode(url, 0, 200);
-            // Simhash that matches in first 16 bits (0xAAAA)
-            const hash = BigInt(0xAAAA) | (BigInt(i) << 16n);
-            graph.updateNodeData(url, { simhash: hash.toString() });
-        }
-
-        for (let i = 0; i < 5; i++) {
-            const url = `https://example.com/g2/${i}`;
-            graph.addNode(url, 0, 200);
-            // Simhash that matches in second 16 bits (0xBBBB << 16)
-            const hash = (BigInt(0xBBBB) << 16n) | BigInt(i);
-            graph.updateNodeData(url, { simhash: hash.toString() });
-        }
-
-        const clusters = detectContentClusters(graph, 5, 3);
-        expect(clusters.length).toBe(2);
-        expect(clusters[0].count).toBe(5);
-        expect(clusters[1].count).toBe(5);
-    });
-});

package/tests/clustering_risk.test.ts

@@ -1,118 +0,0 @@
-import { describe, it, expect, beforeEach } from 'vitest';
-import { Graph } from '../src/graph/graph.js';
-import { detectContentClusters } from '../src/graph/cluster.js';
-
-describe('Cluster Risk Heuristic', () => {
-    let graph: Graph;
-
-    beforeEach(() => {
-        graph = new Graph();
-    });
-
-    it('should assign HIGH risk to clusters with identical titles', () => {
-        const html = '<html><head><title>Duplicate Title</title></head><body>Content</body></html>';
-        const h = 0b101010n.toString();
-
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-        graph.addNode('https://example.com/p3', 0, 200);
-
-        graph.updateNodeData('https://example.com/p1', { simhash: h, html });
-        graph.updateNodeData('https://example.com/p2', { simhash: h, html });
-        graph.updateNodeData('https://example.com/p3', { simhash: h, html });
-
-        const clusters = detectContentClusters(graph, 2, 2);
-
-        expect(clusters.length).toBe(1);
-        expect(clusters[0].risk).toBe('high');
-    });
-
-    it('should assign HIGH risk to clusters with identical H1s', () => {
-        const h = 0b101010n.toString();
-
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-        graph.addNode('https://example.com/p3', 0, 200);
-
-        // Different titles, same H1
-        graph.updateNodeData('https://example.com/p1', {
-            simhash: h,
-            html: '<html><head><title>Page 1</title></head><body><h1>Duplicate Header</h1></body></html>'
-        });
-        graph.updateNodeData('https://example.com/p2', {
-            simhash: h,
-            html: '<html><head><title>Page 2</title></head><body><h1>Duplicate Header</h1></body></html>'
-        });
-        graph.updateNodeData('https://example.com/p3', {
-            simhash: h,
-            html: '<html><head><title>Page 3</title></head><body><h1>Duplicate Header</h1></body></html>'
-        });
-
-        const clusters = detectContentClusters(graph, 2, 2);
-
-        expect(clusters.length).toBe(1);
-        expect(clusters[0].risk).toBe('high');
-    });
-
-    it('should assign LOW risk to small clusters with unique titles and H1s', () => {
-        const h = 0b101010n.toString();
-
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-        graph.addNode('https://example.com/p3', 0, 200);
-
-        graph.updateNodeData('https://example.com/p1', {
-            simhash: h,
-            html: '<html><head><title>Page 1</title></head><body><h1>Header 1</h1></body></html>'
-        });
-        graph.updateNodeData('https://example.com/p2', {
-            simhash: h,
-            html: '<html><head><title>Page 2</title></head><body><h1>Header 2</h1></body></html>'
-        });
-        graph.updateNodeData('https://example.com/p3', {
-            simhash: h,
-            html: '<html><head><title>Page 3</title></head><body><h1>Header 3</h1></body></html>'
-        });
-
-        const clusters = detectContentClusters(graph, 2, 2);
-
-        expect(clusters.length).toBe(1);
-        expect(clusters[0].risk).toBe('low');
-    });
-
-    it('should assign MEDIUM risk to large clusters even with unique titles', () => {
-        const h = 0b101010n.toString();
-
-        // 12 nodes, all unique titles
-        for (let i = 0; i < 12; i++) {
-            const url = `https://example.com/p${i}`;
-            graph.addNode(url, 0, 200);
-            graph.updateNodeData(url, {
-                simhash: h,
-                html: `<html><head><title>Page ${i}</title></head><body><h1>Header ${i}</h1></body></html>`
-            });
-        }
-
-        const clusters = detectContentClusters(graph, 2, 2);
-
-        expect(clusters.length).toBe(1);
-        expect(clusters[0].risk).toBe('medium');
-    });
-
-    it('should handle missing HTML gracefully', () => {
-         const h = 0b101010n.toString();
-
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-
-        // No HTML provided
-        graph.updateNodeData('https://example.com/p1', { simhash: h });
-        graph.updateNodeData('https://example.com/p2', { simhash: h });
-
-        const clusters = detectContentClusters(graph, 2, 2);
-
-        expect(clusters.length).toBe(1);
-        // Fallback to size based? 2 nodes -> low risk
-        expect(clusters[0].risk).toBe('low');
-    });
-});