@crawlith/core 0.1.0

package/tests/proxy_safety.test.ts

@@ -0,0 +1,57 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { Fetcher } from '../src/crawler/fetcher.js';
+import { request, ProxyAgent } from 'undici';
+
+vi.mock('undici', async (importOriginal) => {
+    const original = await importOriginal<typeof import('undici')>();
+    return {
+        ...original,
+        request: vi.fn(),
+        ProxyAgent: vi.fn(function () {
+            return {
+                request: vi.fn(),
+                close: vi.fn()
+            };
+        })
+    };
+});
+
+describe('Proxy Integration', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+
+    it('should use ProxyAgent when proxyUrl is provided', async () => {
+        const fetcher = new Fetcher({ proxyUrl: 'http://proxy.com:8080', rate: 100 });
+        const mockRequest = vi.mocked(request);
+
+        // Mock the request to return a successful response immediately
+        mockRequest.mockResolvedValueOnce({
+            statusCode: 200,
+            headers: {},
+            body: {
+                on: vi.fn((event, cb) => {
+                    if (event === 'data') {
+                        // Simulate async data chunk
+                        setTimeout(() => cb(Buffer.from('ok')), 0);
+                    }
+                    if (event === 'end') {
+                        // Simulate async end
+                        setTimeout(() => cb(), 0);
+                    }
+                    return { on: vi.fn() }; // chaining
+                }),
+                dump: vi.fn(),
+                text: vi.fn().mockResolvedValue('ok')
+            }
+        } as any);
+
+        await fetcher.fetch('http://target.com');
+
+        expect(ProxyAgent).toHaveBeenCalledWith('http://proxy.com:8080');
+    });
+
+    it('should fail fast on invalid proxy URL', () => {
+        expect(() => new Fetcher({ proxyUrl: 'not-a-url' })).toThrow('Invalid proxy URL');
+    });
+});

package/tests/redirect_safety.test.ts

@@ -0,0 +1,73 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { RedirectController } from '../src/core/network/redirectController.js';
+import { Fetcher } from '../src/crawler/fetcher.js';
+import { request } from 'undici';
+
+vi.mock('undici', () => ({
+    request: vi.fn(),
+    ProxyAgent: vi.fn().mockImplementation(() => ({ dispatcher: {} }))
+}));
+
+describe('RedirectController', () => {
+    it('should limit hops', () => {
+        const ctrl = new RedirectController(2);
+        expect(ctrl.nextHop('http://b.com')).toBe(null);
+        expect(ctrl.nextHop('http://c.com')).toBe(null);
+        expect(ctrl.nextHop('http://d.com')).toBe('redirect_limit_exceeded');
+    });
+
+    it('should detect loops', () => {
+        const ctrl = new RedirectController(5);
+        expect(ctrl.nextHop('http://b.com')).toBe(null);
+        expect(ctrl.nextHop('http://a.com')).toBe(null);
+        expect(ctrl.nextHop('http://b.com')).toBe('redirect_loop');
+    });
+});
+
+describe('Fetcher Redirect Integration', () => {
+    let fetcher: Fetcher;
+
+    beforeEach(() => {
+        vi.clearAllMocks();
+        fetcher = new Fetcher({ rate: 100, maxRedirects: 2 });
+    });
+
+    it('should stop at max redirects', async () => {
+        const mockRequest = vi.mocked(request);
+
+        // Return 301 with unique locations
+        mockRequest
+            .mockResolvedValueOnce({
+                statusCode: 301,
+                headers: { location: 'http://a.com' },
+                body: { dump: vi.fn().mockResolvedValue(undefined) }
+            } as any)
+            .mockResolvedValueOnce({
+                statusCode: 301,
+                headers: { location: 'http://b.com' },
+                body: { dump: vi.fn().mockResolvedValue(undefined) }
+            } as any)
+            .mockResolvedValueOnce({
+                statusCode: 301,
+                headers: { location: 'http://c.com' },
+                body: { dump: vi.fn().mockResolvedValue(undefined) }
+            } as any);
+
+        const res = await fetcher.fetch('http://start.com');
+        expect(res.status).toBe('redirect_limit_exceeded');
+        expect(res.redirectChain).toHaveLength(2);
+    });
+
+    it('should detect loops in fetch', async () => {
+        const mockRequest = vi.mocked(request);
+
+        mockRequest.mockResolvedValue({
+            statusCode: 301,
+            headers: { location: 'http://start.com' },
+            body: { dump: vi.fn().mockResolvedValue(undefined) }
+        } as any);
+
+        const res = await fetcher.fetch('http://start.com');
+        expect(res.status).toBe('redirect_loop');
+    });
+});

package/tests/safety.test.ts

@@ -0,0 +1,114 @@
+import { describe, it, expect, vi } from 'vitest';
+import { IPGuard } from '../src/core/security/ipGuard.js';
+import { RateLimiter } from '../src/core/network/rateLimiter.js';
+import { RetryPolicy } from '../src/core/network/retryPolicy.js';
+import { ResponseLimiter } from '../src/core/network/responseLimiter.js';
+import { Readable } from 'stream';
+import * as dns from 'dns';
+
+vi.mock('dns', () => ({
+    resolve4: vi.fn(),
+    resolve6: vi.fn(),
+}));
+
+describe('IPGuard', () => {
+    it('should block IPv4 internal ranges', () => {
+        expect(IPGuard.isInternal('127.0.0.1')).toBe(true);
+        expect(IPGuard.isInternal('10.0.0.1')).toBe(true);
+        expect(IPGuard.isInternal('192.168.1.1')).toBe(true);
+        expect(IPGuard.isInternal('172.16.0.1')).toBe(true);
+        expect(IPGuard.isInternal('172.31.255.255')).toBe(true);
+        expect(IPGuard.isInternal('169.254.1.1')).toBe(true);
+        expect(IPGuard.isInternal('0.0.0.0')).toBe(true);
+    });
+
+    it('should allow public IPv4', () => {
+        expect(IPGuard.isInternal('8.8.8.8')).toBe(false);
+        expect(IPGuard.isInternal('1.1.1.1')).toBe(false);
+        expect(IPGuard.isInternal('172.32.0.1')).toBe(false);
+    });
+
+    it('should block IPv6 internal/local addresses', () => {
+        expect(IPGuard.isInternal('::1')).toBe(true);
+        expect(IPGuard.isInternal('fc00::1')).toBe(true);
+        expect(IPGuard.isInternal('fe80::1')).toBe(true);
+    });
+
+    it('should validate hostname by resolving IPs', async () => {
+        const resolve4Spy = vi.mocked(dns.resolve4);
+        const resolve6Spy = vi.mocked(dns.resolve6);
+
+        resolve4Spy.mockImplementation((_h: string, cb: any) => cb(null, ['1.1.1.1']));
+        resolve6Spy.mockImplementation((_h: string, cb: any) => cb(null, []));
+        expect(await IPGuard.validateHost('example.com')).toBe(true);
+
+        resolve4Spy.mockImplementation((_h: string, cb: any) => cb(null, ['127.0.0.1']));
+        expect(await IPGuard.validateHost('localhost')).toBe(false);
+    });
+});
+
+describe('RateLimiter', () => {
+    it('should enforce rate limits', async () => {
+        const limiter = new RateLimiter(1); // 1 req/sec = 1000ms interval
+        const start = Date.now();
+
+        await limiter.waitForToken('host1'); // returns immediately, tokens becomes 0
+        await limiter.waitForToken('host1'); // waits for refill (1s)
+
+        const elapsed = Date.now() - start;
+        expect(elapsed).toBeGreaterThanOrEqual(1000);
+    }, 5000);
+
+    it('should have separate buckets for hosts', async () => {
+        const limiter = new RateLimiter(1);
+        const start = Date.now();
+
+        await limiter.waitForToken('host1');
+        await limiter.waitForToken('host2');
+
+        const elapsed = Date.now() - start;
+        expect(elapsed).toBeLessThan(100);
+    });
+
+    it('should respect crawlDelay if higher than rate', async () => {
+        const limiter = new RateLimiter(1); // 1000ms interval
+        const start = Date.now();
+
+        await limiter.waitForToken('host3'); // returns immediately, tokens = 0
+        await limiter.waitForToken('host3', 1); // 1s crawl delay
+
+        const elapsed = Date.now() - start;
+        expect(elapsed).toBeGreaterThanOrEqual(1000);
+    }, 5000);
+});
+
+describe('RetryPolicy', () => {
+    it('should retry transient failures', async () => {
+        let calls = 0;
+        const result = await RetryPolicy.execute(
+            async () => {
+                calls++;
+                if (calls < 3) throw new Error('Status 500');
+                return 'success';
+            },
+            (err) => err.message === 'Status 500',
+            { maxRetries: 3, baseDelay: 10 }
+        );
+
+        expect(result).toBe('success');
+        expect(calls).toBe(3);
+    });
+});
+
+describe('ResponseLimiter', () => {
+    it('should stream to string', async () => {
+        const stream = Readable.from(['hello ', 'world']);
+        const result = await ResponseLimiter.streamToString(stream, 100);
+        expect(result).toBe('hello world');
+    });
+
+    it('should abort if limit exceeded', async () => {
+        const stream = Readable.from(['too ', 'large ', 'content']);
+        await expect(ResponseLimiter.streamToString(stream, 5)).rejects.toThrow('Oversized response');
+    });
+});

package/tests/scope.test.ts

@@ -0,0 +1,66 @@
+import { describe, it, expect } from 'vitest';
+import { DomainFilter } from '../src/core/scope/domainFilter.js';
+import { SubdomainPolicy } from '../src/core/scope/subdomainPolicy.js';
+import { ScopeManager } from '../src/core/scope/scopeManager.js';
+
+describe('DomainFilter', () => {
+    it('should normalize hostnames', () => {
+        const filter = new DomainFilter(['EXAMPLE.COM.'], ['DENY.COM.']);
+        expect(filter.isAllowed('example.com')).toBe(true);
+        expect(filter.isAllowed('deny.com')).toBe(false);
+    });
+
+    it('should respect precedence (deny wins)', () => {
+        const filter = new DomainFilter(['example.com'], ['example.com']);
+        expect(filter.isAllowed('example.com')).toBe(false);
+    });
+
+    it('should handle punycode', () => {
+        // xn--80ak6aa92e.com is punycode for пример.com
+        const filter = new DomainFilter(['xn--80ak6aa92e.com']);
+        expect(filter.isAllowed('XN--80AK6AA92E.COM')).toBe(true);
+    });
+
+    it('should block if not in allow list (when list not empty)', () => {
+        const filter = new DomainFilter(['allowed.com']);
+        expect(filter.isAllowed('other.com')).toBe(false);
+    });
+});
+
+describe('SubdomainPolicy', () => {
+    it('should enforce exact match by default', () => {
+        const policy = new SubdomainPolicy('https://example.com');
+        expect(policy.isAllowed('example.com')).toBe(true);
+        expect(policy.isAllowed('sub.example.com')).toBe(false);
+    });
+
+    it('should allow valid subdomains when enabled', () => {
+        const policy = new SubdomainPolicy('https://example.com', true);
+        expect(policy.isAllowed('example.com')).toBe(true);
+        expect(policy.isAllowed('sub.example.com')).toBe(true);
+        expect(policy.isAllowed('deep.sub.example.com')).toBe(true);
+    });
+
+    it('should reject malicious suffix matches', () => {
+        const policy = new SubdomainPolicy('https://example.com', true);
+        expect(policy.isAllowed('evil-example.com')).toBe(false);
+        expect(policy.isAllowed('example.com.evil.com')).toBe(false);
+    });
+});
+
+describe('ScopeManager', () => {
+    it('should compose policies correctly', () => {
+        const manager = new ScopeManager({
+            rootUrl: 'https://example.com',
+            allowedDomains: ['example.com', 'sub.example.com', 'other.com'],
+            deniedDomains: ['bad.example.com'],
+            includeSubdomains: true
+        });
+
+        expect(manager.isUrlEligible('https://example.com/')).toBe('allowed');
+        expect(manager.isUrlEligible('https://sub.example.com/')).toBe('allowed');
+        expect(manager.isUrlEligible('https://bad.example.com/')).toBe('blocked_by_domain_filter');
+        expect(manager.isUrlEligible('https://other.com/')).toBe('allowed');
+        expect(manager.isUrlEligible('https://google.com/')).toBe('blocked_by_domain_filter');
+    });
+});

package/tests/scoring.test.ts

@@ -0,0 +1,59 @@
+import { expect, test } from 'vitest';
+import { scorePageSeo, aggregateSiteScore } from '../src/analysis/scoring.js';
+import { PageAnalysis } from '../src/analysis/analyze.js';
+
+const basePage: PageAnalysis = {
+  url: 'https://example.com',
+  status: 200,
+  title: { value: 'x'.repeat(55), length: 55, status: 'ok' },
+  metaDescription: { value: 'x'.repeat(150), length: 150, status: 'ok' },
+  h1: { count: 1, status: 'ok', matchesTitle: false },
+  content: { wordCount: 700, textHtmlRatio: 0.3, uniqueSentenceCount: 8 },
+  thinScore: 0,
+  images: { totalImages: 2, missingAlt: 0, emptyAlt: 0 },
+  links: { internalLinks: 5, externalLinks: 2, nofollowCount: 1, externalRatio: 2 / 7 },
+  structuredData: { present: true, valid: true, types: ['Article'] },
+  seoScore: 0
+};
+
+test('page score stays in 0-100', () => {
+  expect(scorePageSeo(basePage)).toBeGreaterThanOrEqual(0);
+  expect(scorePageSeo(basePage)).toBeLessThanOrEqual(100);
+
+  const badPage: PageAnalysis = {
+    ...basePage,
+    title: { value: null, length: 0, status: 'missing' },
+    metaDescription: { value: null, length: 0, status: 'missing' },
+    h1: { count: 0, status: 'critical', matchesTitle: false },
+    content: { wordCount: 0, textHtmlRatio: 0, uniqueSentenceCount: 0 },
+    thinScore: 100,
+    images: { totalImages: 2, missingAlt: 2, emptyAlt: 0 },
+    structuredData: { present: false, valid: false, types: [] },
+    links: { internalLinks: 0, externalLinks: 9, nofollowCount: 9, externalRatio: 1 }
+  };
+  expect(scorePageSeo(badPage)).toBeLessThan(50);
+});
+
+test('aggregate site score includes existing metrics signals', () => {
+  const score = aggregateSiteScore({
+    totalPages: 2,
+    totalEdges: 1,
+    orphanPages: ['https://example.com/x'],
+    nearOrphans: [],
+    deepPages: [],
+    topAuthorityPages: [{ url: 'a', authority: 1 }],
+    averageOutDegree: 1,
+    maxDepthFound: 1,
+    crawlEfficiencyScore: 0.8,
+    averageDepth: 1,
+    structuralEntropy: 2,
+    limitReached: false
+  }, [
+    { ...basePage, seoScore: 70 },
+    { ...basePage, seoScore: 90, url: 'https://example.com/2' }
+  ]);
+
+  expect(score.seoHealthScore).toBe(80);
+  expect(score.overallScore).toBeGreaterThan(0);
+  expect(score.overallScore).toBeLessThanOrEqual(100);
+});

package/tests/sitemap.test.ts

@@ -0,0 +1,88 @@
+import { test, expect, beforeEach } from 'vitest';
+import { Sitemap } from '../src/crawler/sitemap.js';
+import { MockAgent, setGlobalDispatcher } from 'undici';
+
+let mockAgent: MockAgent;
+
+beforeEach(() => {
+  mockAgent = new MockAgent();
+  mockAgent.disableNetConnect();
+  setGlobalDispatcher(mockAgent);
+});
+
+test('fetches and parses simple sitemap', async () => {
+  const client = mockAgent.get('https://example.com');
+  client.intercept({
+    path: '/sitemap.xml',
+    method: 'GET'
+  }).reply(200, `
+    <?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+      <url>
+        <loc>https://example.com/page1</loc>
+      </url>
+      <url>
+        <loc>https://example.com/page2</loc>
+      </url>
+    </urlset>
+  `);
+
+  const sitemap = new Sitemap();
+  const urls = await sitemap.fetch('https://example.com/sitemap.xml');
+  expect(urls).toContain('https://example.com/page1');
+  expect(urls).toContain('https://example.com/page2');
+  expect(urls.length).toBe(2);
+});
+
+test('handles sitemap index recursively', async () => {
+  const client = mockAgent.get('https://example.com');
+
+  // Index
+  client.intercept({
+    path: '/sitemap-index.xml',
+    method: 'GET'
+  }).reply(200, `
+    <?xml version="1.0" encoding="UTF-8"?>
+    <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+      <sitemap>
+        <loc>https://example.com/sitemap1.xml</loc>
+      </sitemap>
+    </sitemapindex>
+  `);
+
+  // Child sitemap
+  client.intercept({
+    path: '/sitemap1.xml',
+    method: 'GET'
+  }).reply(200, `
+    <?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+      <url>
+        <loc>https://example.com/page3</loc>
+      </url>
+    </urlset>
+  `);
+
+  const sitemap = new Sitemap();
+  const urls = await sitemap.fetch('https://example.com/sitemap-index.xml');
+  expect(urls).toContain('https://example.com/page3');
+  expect(urls.length).toBe(1);
+});
+
+test('handles invalid xml gracefully', async () => {
+  const client = mockAgent.get('https://example.com');
+  client.intercept({ path: '/bad.xml', method: 'GET' }).reply(200, 'Not XML');
+
+  const sitemap = new Sitemap();
+  const urls = await sitemap.fetch('https://example.com/bad.xml');
+  expect(urls.length).toBe(0);
+});
+
+test('handles fetch errors gracefully', async () => {
+  const client = mockAgent.get('https://example.com');
+  client.intercept({ path: '/error.xml', method: 'GET' }).reply(500, 'Error');
+
+  const sitemap = new Sitemap();
+  const urls = await sitemap.fetch('https://example.com/error.xml');
+  expect(urls.length).toBe(0);
+});

package/tests/soft404.test.ts

@@ -0,0 +1,41 @@
+import { describe, it, expect } from 'vitest';
+import { Parser } from '../src/crawler/parser.js';
+
+describe('Soft 404 Detection', () => {
+    const parser = new Parser();
+    const baseUrl = 'https://example.com';
+
+    it('should detect soft 404 by title pattern', () => {
+        const html = '<html><head><title>Page Not Found</title></head><body>Welcome to the site</body></html>';
+        const result = parser.parse(html, baseUrl, 200);
+        expect(result.soft404Score).toBeGreaterThan(0.3);
+        expect(result.soft404Signals).toContain('title_pattern_not_found');
+    });
+
+    it('should detect soft 404 by H1 pattern', () => {
+        const html = '<html><body><h1>404 Error</h1></body></html>';
+        const result = parser.parse(html, baseUrl, 200);
+        expect(result.soft404Score).toBeGreaterThan(0.2);
+        expect(result.soft404Signals).toContain('h1_pattern_404');
+    });
+
+    it('should detect soft 404 by very low word count', () => {
+        const html = '<html><body>Short text</body></html>';
+        const result = parser.parse(html, baseUrl, 200);
+        expect(result.soft404Score).toBeGreaterThan(0.2);
+        expect(result.soft404Signals).toContain('very_low_word_count');
+    });
+
+    it('should detect soft 404 by lack of outbound links', () => {
+        const html = '<html><body>A page with some text but no links.</body></html>';
+        const result = parser.parse(html, baseUrl, 200);
+        expect(result.soft404Signals).toContain('no_outbound_links');
+    });
+
+    it('should combine multiple signals for high score', () => {
+        const html = '<html><head><title>Error</title></head><body><h1>Not Found</h1><p>The requested page was not found.</p></body></html>';
+        const result = parser.parse(html, baseUrl, 200);
+        // title (0.4) + h1 (0.3) + body phrase (0.2) + low word count (0.3) = 1.2 -> capped at 1.0
+        expect(result.soft404Score).toBe(1.0);
+    });
+});

package/tests/trap.test.ts

@@ -0,0 +1,39 @@
+import { describe, it, expect } from 'vitest';
+import { TrapDetector } from '../src/crawler/trap.js';
+
+describe('TrapDetector', () => {
+    const detector = new TrapDetector();
+
+    it('should detect session ID traps', () => {
+        const result = detector.checkTrap('https://example.com/page?sid=12345', 1);
+        expect(result.risk).toBeGreaterThan(0.8);
+        expect(result.type).toBe('session_trap');
+    });
+
+    it('should detect calendar patterns', () => {
+        const result = detector.checkTrap('https://example.com/archive/2023/12/01/', 1);
+        expect(result.risk).toBeGreaterThan(0.6);
+        expect(result.type).toBe('calendar_trap');
+    });
+
+    it('should detect pagination loops', () => {
+        // Simulate many pages
+        for (let i = 1; i <= 60; i++) {
+            detector.checkTrap(`https://example.com/blog?page=${i}`, 1);
+        }
+        const result = detector.checkTrap('https://example.com/blog?page=61', 1);
+        expect(result.risk).toBeGreaterThan(0.8);
+        expect(result.type).toBe('pagination_loop');
+    });
+
+    it('should detect faceted navigation / parameter explosion', () => {
+        detector.reset();
+        const basePath = 'https://example.com/products';
+        for (let i = 1; i <= 35; i++) {
+            detector.checkTrap(`${basePath}?color=red&size=${i}`, 1);
+        }
+        const result = detector.checkTrap(`${basePath}?color=blue&size=large`, 1);
+        expect(result.risk).toBeGreaterThan(0.9);
+        expect(result.type).toBe('faceted_navigation');
+    });
+});

package/tests/visualization_data.test.ts

@@ -0,0 +1,46 @@
+import { describe, it, expect } from 'vitest';
+import { SITEGRAPH_HTML } from '../src/report/sitegraph_template.js';
+import { Graph } from '../src/graph/graph.js';
+import { computePageRank } from '../src/graph/pagerank.js';
+
+describe('Visualization Data & Template', () => {
+    it('should include pageRankScore in graph JSON output after PageRank computation', () => {
+        const graph = new Graph();
+        graph.addNode('https://a.com', 0, 200);
+        graph.addNode('https://b.com', 1, 200);
+        graph.addEdge('https://a.com', 'https://b.com');
+
+        computePageRank(graph);
+
+        const json = graph.toJSON();
+        const nodeA = json.nodes.find(n => n.url === 'https://a.com');
+        const nodeB = json.nodes.find(n => n.url === 'https://b.com');
+
+        expect(nodeA).toBeDefined();
+        expect(nodeB).toBeDefined();
+        expect(typeof nodeA?.pageRankScore).toBe('number');
+        expect(typeof nodeB?.pageRankScore).toBe('number');
+    });
+
+    it('should contain UI toggle buttons for Authority Mode', () => {
+        expect(SITEGRAPH_HTML).toContain('id="btn-auth-pagerank"');
+        expect(SITEGRAPH_HTML).toContain('id="btn-auth-structural"');
+    });
+
+    it('should contain setAuthorityMode function', () => {
+        // Use regex to be flexible with whitespace
+        expect(SITEGRAPH_HTML).toMatch(/function\s+setAuthorityMode\s*\(mode,\s*btn\)/);
+        expect(SITEGRAPH_HTML).toContain('n.authority = mode === \'pagerank\' ? n.pageRankAuthority : n.structuralAuthority');
+    });
+
+    it('should contain logic to calculate pageRankAuthority from pageRankScore', () => {
+        expect(SITEGRAPH_HTML).toContain('n.pageRankAuthority = n.pageRankScore / 100');
+        expect(SITEGRAPH_HTML).toContain('n.structuralAuthority = Math.log(1 + n.inLinks)');
+    });
+
+    it('should update details panel to show both metrics', () => {
+        expect(SITEGRAPH_HTML).toContain('id="d-auth-container"');
+        expect(SITEGRAPH_HTML).toContain('In-Degree: ${structVal}');
+        expect(SITEGRAPH_HTML).toContain('PR: <strong>${prVal}</strong>');
+    });
+});

package/tsconfig.json

@@ -0,0 +1,11 @@
+{
+    "extends": "../../tsconfig.json",
+    "compilerOptions": {
+        "outDir": "dist",
+        "rootDir": "src",
+        "declaration": true
+    },
+    "include": [
+        "src"
+    ]
+}