@crawlith/core 0.1.0 → 0.1.2

package/tests/proxy_safety.test.ts

@@ -1,57 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { Fetcher } from '../src/crawler/fetcher.js';
-import { request, ProxyAgent } from 'undici';
-
-vi.mock('undici', async (importOriginal) => {
-    const original = await importOriginal<typeof import('undici')>();
-    return {
-        ...original,
-        request: vi.fn(),
-        ProxyAgent: vi.fn(function () {
-            return {
-                request: vi.fn(),
-                close: vi.fn()
-            };
-        })
-    };
-});
-
-describe('Proxy Integration', () => {
-    beforeEach(() => {
-        vi.clearAllMocks();
-    });
-
-    it('should use ProxyAgent when proxyUrl is provided', async () => {
-        const fetcher = new Fetcher({ proxyUrl: 'http://proxy.com:8080', rate: 100 });
-        const mockRequest = vi.mocked(request);
-
-        // Mock the request to return a successful response immediately
-        mockRequest.mockResolvedValueOnce({
-            statusCode: 200,
-            headers: {},
-            body: {
-                on: vi.fn((event, cb) => {
-                    if (event === 'data') {
-                        // Simulate async data chunk
-                        setTimeout(() => cb(Buffer.from('ok')), 0);
-                    }
-                    if (event === 'end') {
-                        // Simulate async end
-                        setTimeout(() => cb(), 0);
-                    }
-                    return { on: vi.fn() }; // chaining
-                }),
-                dump: vi.fn(),
-                text: vi.fn().mockResolvedValue('ok')
-            }
-        } as any);
-
-        await fetcher.fetch('http://target.com');
-
-        expect(ProxyAgent).toHaveBeenCalledWith('http://proxy.com:8080');
-    });
-
-    it('should fail fast on invalid proxy URL', () => {
-        expect(() => new Fetcher({ proxyUrl: 'not-a-url' })).toThrow('Invalid proxy URL');
-    });
-});

package/tests/redirect_safety.test.ts

@@ -1,73 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { RedirectController } from '../src/core/network/redirectController.js';
-import { Fetcher } from '../src/crawler/fetcher.js';
-import { request } from 'undici';
-
-vi.mock('undici', () => ({
-    request: vi.fn(),
-    ProxyAgent: vi.fn().mockImplementation(() => ({ dispatcher: {} }))
-}));
-
-describe('RedirectController', () => {
-    it('should limit hops', () => {
-        const ctrl = new RedirectController(2);
-        expect(ctrl.nextHop('http://b.com')).toBe(null);
-        expect(ctrl.nextHop('http://c.com')).toBe(null);
-        expect(ctrl.nextHop('http://d.com')).toBe('redirect_limit_exceeded');
-    });
-
-    it('should detect loops', () => {
-        const ctrl = new RedirectController(5);
-        expect(ctrl.nextHop('http://b.com')).toBe(null);
-        expect(ctrl.nextHop('http://a.com')).toBe(null);
-        expect(ctrl.nextHop('http://b.com')).toBe('redirect_loop');
-    });
-});
-
-describe('Fetcher Redirect Integration', () => {
-    let fetcher: Fetcher;
-
-    beforeEach(() => {
-        vi.clearAllMocks();
-        fetcher = new Fetcher({ rate: 100, maxRedirects: 2 });
-    });
-
-    it('should stop at max redirects', async () => {
-        const mockRequest = vi.mocked(request);
-
-        // Return 301 with unique locations
-        mockRequest
-            .mockResolvedValueOnce({
-                statusCode: 301,
-                headers: { location: 'http://a.com' },
-                body: { dump: vi.fn().mockResolvedValue(undefined) }
-            } as any)
-            .mockResolvedValueOnce({
-                statusCode: 301,
-                headers: { location: 'http://b.com' },
-                body: { dump: vi.fn().mockResolvedValue(undefined) }
-            } as any)
-            .mockResolvedValueOnce({
-                statusCode: 301,
-                headers: { location: 'http://c.com' },
-                body: { dump: vi.fn().mockResolvedValue(undefined) }
-            } as any);
-
-        const res = await fetcher.fetch('http://start.com');
-        expect(res.status).toBe('redirect_limit_exceeded');
-        expect(res.redirectChain).toHaveLength(2);
-    });
-
-    it('should detect loops in fetch', async () => {
-        const mockRequest = vi.mocked(request);
-
-        mockRequest.mockResolvedValue({
-            statusCode: 301,
-            headers: { location: 'http://start.com' },
-            body: { dump: vi.fn().mockResolvedValue(undefined) }
-        } as any);
-
-        const res = await fetcher.fetch('http://start.com');
-        expect(res.status).toBe('redirect_loop');
-    });
-});

package/tests/safety.test.ts

@@ -1,114 +0,0 @@
-import { describe, it, expect, vi } from 'vitest';
-import { IPGuard } from '../src/core/security/ipGuard.js';
-import { RateLimiter } from '../src/core/network/rateLimiter.js';
-import { RetryPolicy } from '../src/core/network/retryPolicy.js';
-import { ResponseLimiter } from '../src/core/network/responseLimiter.js';
-import { Readable } from 'stream';
-import * as dns from 'dns';
-
-vi.mock('dns', () => ({
-    resolve4: vi.fn(),
-    resolve6: vi.fn(),
-}));
-
-describe('IPGuard', () => {
-    it('should block IPv4 internal ranges', () => {
-        expect(IPGuard.isInternal('127.0.0.1')).toBe(true);
-        expect(IPGuard.isInternal('10.0.0.1')).toBe(true);
-        expect(IPGuard.isInternal('192.168.1.1')).toBe(true);
-        expect(IPGuard.isInternal('172.16.0.1')).toBe(true);
-        expect(IPGuard.isInternal('172.31.255.255')).toBe(true);
-        expect(IPGuard.isInternal('169.254.1.1')).toBe(true);
-        expect(IPGuard.isInternal('0.0.0.0')).toBe(true);
-    });
-
-    it('should allow public IPv4', () => {
-        expect(IPGuard.isInternal('8.8.8.8')).toBe(false);
-        expect(IPGuard.isInternal('1.1.1.1')).toBe(false);
-        expect(IPGuard.isInternal('172.32.0.1')).toBe(false);
-    });
-
-    it('should block IPv6 internal/local addresses', () => {
-        expect(IPGuard.isInternal('::1')).toBe(true);
-        expect(IPGuard.isInternal('fc00::1')).toBe(true);
-        expect(IPGuard.isInternal('fe80::1')).toBe(true);
-    });
-
-    it('should validate hostname by resolving IPs', async () => {
-        const resolve4Spy = vi.mocked(dns.resolve4);
-        const resolve6Spy = vi.mocked(dns.resolve6);
-
-        resolve4Spy.mockImplementation((_h: string, cb: any) => cb(null, ['1.1.1.1']));
-        resolve6Spy.mockImplementation((_h: string, cb: any) => cb(null, []));
-        expect(await IPGuard.validateHost('example.com')).toBe(true);
-
-        resolve4Spy.mockImplementation((_h: string, cb: any) => cb(null, ['127.0.0.1']));
-        expect(await IPGuard.validateHost('localhost')).toBe(false);
-    });
-});
-
-describe('RateLimiter', () => {
-    it('should enforce rate limits', async () => {
-        const limiter = new RateLimiter(1); // 1 req/sec = 1000ms interval
-        const start = Date.now();
-
-        await limiter.waitForToken('host1'); // returns immediately, tokens becomes 0
-        await limiter.waitForToken('host1'); // waits for refill (1s)
-
-        const elapsed = Date.now() - start;
-        expect(elapsed).toBeGreaterThanOrEqual(1000);
-    }, 5000);
-
-    it('should have separate buckets for hosts', async () => {
-        const limiter = new RateLimiter(1);
-        const start = Date.now();
-
-        await limiter.waitForToken('host1');
-        await limiter.waitForToken('host2');
-
-        const elapsed = Date.now() - start;
-        expect(elapsed).toBeLessThan(100);
-    });
-
-    it('should respect crawlDelay if higher than rate', async () => {
-        const limiter = new RateLimiter(1); // 1000ms interval
-        const start = Date.now();
-
-        await limiter.waitForToken('host3'); // returns immediately, tokens = 0
-        await limiter.waitForToken('host3', 1); // 1s crawl delay
-
-        const elapsed = Date.now() - start;
-        expect(elapsed).toBeGreaterThanOrEqual(1000);
-    }, 5000);
-});
-
-describe('RetryPolicy', () => {
-    it('should retry transient failures', async () => {
-        let calls = 0;
-        const result = await RetryPolicy.execute(
-            async () => {
-                calls++;
-                if (calls < 3) throw new Error('Status 500');
-                return 'success';
-            },
-            (err) => err.message === 'Status 500',
-            { maxRetries: 3, baseDelay: 10 }
-        );
-
-        expect(result).toBe('success');
-        expect(calls).toBe(3);
-    });
-});
-
-describe('ResponseLimiter', () => {
-    it('should stream to string', async () => {
-        const stream = Readable.from(['hello ', 'world']);
-        const result = await ResponseLimiter.streamToString(stream, 100);
-        expect(result).toBe('hello world');
-    });
-
-    it('should abort if limit exceeded', async () => {
-        const stream = Readable.from(['too ', 'large ', 'content']);
-        await expect(ResponseLimiter.streamToString(stream, 5)).rejects.toThrow('Oversized response');
-    });
-});

package/tests/scope.test.ts

@@ -1,66 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { DomainFilter } from '../src/core/scope/domainFilter.js';
-import { SubdomainPolicy } from '../src/core/scope/subdomainPolicy.js';
-import { ScopeManager } from '../src/core/scope/scopeManager.js';
-
-describe('DomainFilter', () => {
-    it('should normalize hostnames', () => {
-        const filter = new DomainFilter(['EXAMPLE.COM.'], ['DENY.COM.']);
-        expect(filter.isAllowed('example.com')).toBe(true);
-        expect(filter.isAllowed('deny.com')).toBe(false);
-    });
-
-    it('should respect precedence (deny wins)', () => {
-        const filter = new DomainFilter(['example.com'], ['example.com']);
-        expect(filter.isAllowed('example.com')).toBe(false);
-    });
-
-    it('should handle punycode', () => {
-        // xn--80ak6aa92e.com is punycode for пример.com
-        const filter = new DomainFilter(['xn--80ak6aa92e.com']);
-        expect(filter.isAllowed('XN--80AK6AA92E.COM')).toBe(true);
-    });
-
-    it('should block if not in allow list (when list not empty)', () => {
-        const filter = new DomainFilter(['allowed.com']);
-        expect(filter.isAllowed('other.com')).toBe(false);
-    });
-});
-
-describe('SubdomainPolicy', () => {
-    it('should enforce exact match by default', () => {
-        const policy = new SubdomainPolicy('https://example.com');
-        expect(policy.isAllowed('example.com')).toBe(true);
-        expect(policy.isAllowed('sub.example.com')).toBe(false);
-    });
-
-    it('should allow valid subdomains when enabled', () => {
-        const policy = new SubdomainPolicy('https://example.com', true);
-        expect(policy.isAllowed('example.com')).toBe(true);
-        expect(policy.isAllowed('sub.example.com')).toBe(true);
-        expect(policy.isAllowed('deep.sub.example.com')).toBe(true);
-    });
-
-    it('should reject malicious suffix matches', () => {
-        const policy = new SubdomainPolicy('https://example.com', true);
-        expect(policy.isAllowed('evil-example.com')).toBe(false);
-        expect(policy.isAllowed('example.com.evil.com')).toBe(false);
-    });
-});
-
-describe('ScopeManager', () => {
-    it('should compose policies correctly', () => {
-        const manager = new ScopeManager({
-            rootUrl: 'https://example.com',
-            allowedDomains: ['example.com', 'sub.example.com', 'other.com'],
-            deniedDomains: ['bad.example.com'],
-            includeSubdomains: true
-        });
-
-        expect(manager.isUrlEligible('https://example.com/')).toBe('allowed');
-        expect(manager.isUrlEligible('https://sub.example.com/')).toBe('allowed');
-        expect(manager.isUrlEligible('https://bad.example.com/')).toBe('blocked_by_domain_filter');
-        expect(manager.isUrlEligible('https://other.com/')).toBe('allowed');
-        expect(manager.isUrlEligible('https://google.com/')).toBe('blocked_by_domain_filter');
-    });
-});

package/tests/scoring.test.ts

@@ -1,59 +0,0 @@
-import { expect, test } from 'vitest';
-import { scorePageSeo, aggregateSiteScore } from '../src/analysis/scoring.js';
-import { PageAnalysis } from '../src/analysis/analyze.js';
-
-const basePage: PageAnalysis = {
-  url: 'https://example.com',
-  status: 200,
-  title: { value: 'x'.repeat(55), length: 55, status: 'ok' },
-  metaDescription: { value: 'x'.repeat(150), length: 150, status: 'ok' },
-  h1: { count: 1, status: 'ok', matchesTitle: false },
-  content: { wordCount: 700, textHtmlRatio: 0.3, uniqueSentenceCount: 8 },
-  thinScore: 0,
-  images: { totalImages: 2, missingAlt: 0, emptyAlt: 0 },
-  links: { internalLinks: 5, externalLinks: 2, nofollowCount: 1, externalRatio: 2 / 7 },
-  structuredData: { present: true, valid: true, types: ['Article'] },
-  seoScore: 0
-};
-
-test('page score stays in 0-100', () => {
-  expect(scorePageSeo(basePage)).toBeGreaterThanOrEqual(0);
-  expect(scorePageSeo(basePage)).toBeLessThanOrEqual(100);
-
-  const badPage: PageAnalysis = {
-    ...basePage,
-    title: { value: null, length: 0, status: 'missing' },
-    metaDescription: { value: null, length: 0, status: 'missing' },
-    h1: { count: 0, status: 'critical', matchesTitle: false },
-    content: { wordCount: 0, textHtmlRatio: 0, uniqueSentenceCount: 0 },
-    thinScore: 100,
-    images: { totalImages: 2, missingAlt: 2, emptyAlt: 0 },
-    structuredData: { present: false, valid: false, types: [] },
-    links: { internalLinks: 0, externalLinks: 9, nofollowCount: 9, externalRatio: 1 }
-  };
-  expect(scorePageSeo(badPage)).toBeLessThan(50);
-});
-
-test('aggregate site score includes existing metrics signals', () => {
-  const score = aggregateSiteScore({
-    totalPages: 2,
-    totalEdges: 1,
-    orphanPages: ['https://example.com/x'],
-    nearOrphans: [],
-    deepPages: [],
-    topAuthorityPages: [{ url: 'a', authority: 1 }],
-    averageOutDegree: 1,
-    maxDepthFound: 1,
-    crawlEfficiencyScore: 0.8,
-    averageDepth: 1,
-    structuralEntropy: 2,
-    limitReached: false
-  }, [
-    { ...basePage, seoScore: 70 },
-    { ...basePage, seoScore: 90, url: 'https://example.com/2' }
-  ]);
-
-  expect(score.seoHealthScore).toBe(80);
-  expect(score.overallScore).toBeGreaterThan(0);
-  expect(score.overallScore).toBeLessThanOrEqual(100);
-});

package/tests/sitemap.test.ts

@@ -1,88 +0,0 @@
-import { test, expect, beforeEach } from 'vitest';
-import { Sitemap } from '../src/crawler/sitemap.js';
-import { MockAgent, setGlobalDispatcher } from 'undici';
-
-let mockAgent: MockAgent;
-
-beforeEach(() => {
-  mockAgent = new MockAgent();
-  mockAgent.disableNetConnect();
-  setGlobalDispatcher(mockAgent);
-});
-
-test('fetches and parses simple sitemap', async () => {
-  const client = mockAgent.get('https://example.com');
-  client.intercept({
-    path: '/sitemap.xml',
-    method: 'GET'
-  }).reply(200, `
-    <?xml version="1.0" encoding="UTF-8"?>
-    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-      <url>
-        <loc>https://example.com/page1</loc>
-      </url>
-      <url>
-        <loc>https://example.com/page2</loc>
-      </url>
-    </urlset>
-  `);
-
-  const sitemap = new Sitemap();
-  const urls = await sitemap.fetch('https://example.com/sitemap.xml');
-  expect(urls).toContain('https://example.com/page1');
-  expect(urls).toContain('https://example.com/page2');
-  expect(urls.length).toBe(2);
-});
-
-test('handles sitemap index recursively', async () => {
-  const client = mockAgent.get('https://example.com');
-
-  // Index
-  client.intercept({
-    path: '/sitemap-index.xml',
-    method: 'GET'
-  }).reply(200, `
-    <?xml version="1.0" encoding="UTF-8"?>
-    <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-      <sitemap>
-        <loc>https://example.com/sitemap1.xml</loc>
-      </sitemap>
-    </sitemapindex>
-  `);
-
-  // Child sitemap
-  client.intercept({
-    path: '/sitemap1.xml',
-    method: 'GET'
-  }).reply(200, `
-    <?xml version="1.0" encoding="UTF-8"?>
-    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-      <url>
-        <loc>https://example.com/page3</loc>
-      </url>
-    </urlset>
-  `);
-
-  const sitemap = new Sitemap();
-  const urls = await sitemap.fetch('https://example.com/sitemap-index.xml');
-  expect(urls).toContain('https://example.com/page3');
-  expect(urls.length).toBe(1);
-});
-
-test('handles invalid xml gracefully', async () => {
-  const client = mockAgent.get('https://example.com');
-  client.intercept({ path: '/bad.xml', method: 'GET' }).reply(200, 'Not XML');
-
-  const sitemap = new Sitemap();
-  const urls = await sitemap.fetch('https://example.com/bad.xml');
-  expect(urls.length).toBe(0);
-});
-
-test('handles fetch errors gracefully', async () => {
-  const client = mockAgent.get('https://example.com');
-  client.intercept({ path: '/error.xml', method: 'GET' }).reply(500, 'Error');
-
-  const sitemap = new Sitemap();
-  const urls = await sitemap.fetch('https://example.com/error.xml');
-  expect(urls.length).toBe(0);
-});

package/tests/soft404.test.ts

@@ -1,41 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { Parser } from '../src/crawler/parser.js';
-
-describe('Soft 404 Detection', () => {
-    const parser = new Parser();
-    const baseUrl = 'https://example.com';
-
-    it('should detect soft 404 by title pattern', () => {
-        const html = '<html><head><title>Page Not Found</title></head><body>Welcome to the site</body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        expect(result.soft404Score).toBeGreaterThan(0.3);
-        expect(result.soft404Signals).toContain('title_pattern_not_found');
-    });
-
-    it('should detect soft 404 by H1 pattern', () => {
-        const html = '<html><body><h1>404 Error</h1></body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        expect(result.soft404Score).toBeGreaterThan(0.2);
-        expect(result.soft404Signals).toContain('h1_pattern_404');
-    });
-
-    it('should detect soft 404 by very low word count', () => {
-        const html = '<html><body>Short text</body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        expect(result.soft404Score).toBeGreaterThan(0.2);
-        expect(result.soft404Signals).toContain('very_low_word_count');
-    });
-
-    it('should detect soft 404 by lack of outbound links', () => {
-        const html = '<html><body>A page with some text but no links.</body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        expect(result.soft404Signals).toContain('no_outbound_links');
-    });
-
-    it('should combine multiple signals for high score', () => {
-        const html = '<html><head><title>Error</title></head><body><h1>Not Found</h1><p>The requested page was not found.</p></body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        // title (0.4) + h1 (0.3) + body phrase (0.2) + low word count (0.3) = 1.2 -> capped at 1.0
-        expect(result.soft404Score).toBe(1.0);
-    });
-});

package/tests/trap.test.ts

@@ -1,39 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { TrapDetector } from '../src/crawler/trap.js';
-
-describe('TrapDetector', () => {
-    const detector = new TrapDetector();
-
-    it('should detect session ID traps', () => {
-        const result = detector.checkTrap('https://example.com/page?sid=12345', 1);
-        expect(result.risk).toBeGreaterThan(0.8);
-        expect(result.type).toBe('session_trap');
-    });
-
-    it('should detect calendar patterns', () => {
-        const result = detector.checkTrap('https://example.com/archive/2023/12/01/', 1);
-        expect(result.risk).toBeGreaterThan(0.6);
-        expect(result.type).toBe('calendar_trap');
-    });
-
-    it('should detect pagination loops', () => {
-        // Simulate many pages
-        for (let i = 1; i <= 60; i++) {
-            detector.checkTrap(`https://example.com/blog?page=${i}`, 1);
-        }
-        const result = detector.checkTrap('https://example.com/blog?page=61', 1);
-        expect(result.risk).toBeGreaterThan(0.8);
-        expect(result.type).toBe('pagination_loop');
-    });
-
-    it('should detect faceted navigation / parameter explosion', () => {
-        detector.reset();
-        const basePath = 'https://example.com/products';
-        for (let i = 1; i <= 35; i++) {
-            detector.checkTrap(`${basePath}?color=red&size=${i}`, 1);
-        }
-        const result = detector.checkTrap(`${basePath}?color=blue&size=large`, 1);
-        expect(result.risk).toBeGreaterThan(0.9);
-        expect(result.type).toBe('faceted_navigation');
-    });
-});

package/tests/visualization_data.test.ts

@@ -1,46 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { SITEGRAPH_HTML } from '../src/report/sitegraph_template.js';
-import { Graph } from '../src/graph/graph.js';
-import { computePageRank } from '../src/graph/pagerank.js';
-
-describe('Visualization Data & Template', () => {
-    it('should include pageRankScore in graph JSON output after PageRank computation', () => {
-        const graph = new Graph();
-        graph.addNode('https://a.com', 0, 200);
-        graph.addNode('https://b.com', 1, 200);
-        graph.addEdge('https://a.com', 'https://b.com');
-
-        computePageRank(graph);
-
-        const json = graph.toJSON();
-        const nodeA = json.nodes.find(n => n.url === 'https://a.com');
-        const nodeB = json.nodes.find(n => n.url === 'https://b.com');
-
-        expect(nodeA).toBeDefined();
-        expect(nodeB).toBeDefined();
-        expect(typeof nodeA?.pageRankScore).toBe('number');
-        expect(typeof nodeB?.pageRankScore).toBe('number');
-    });
-
-    it('should contain UI toggle buttons for Authority Mode', () => {
-        expect(SITEGRAPH_HTML).toContain('id="btn-auth-pagerank"');
-        expect(SITEGRAPH_HTML).toContain('id="btn-auth-structural"');
-    });
-
-    it('should contain setAuthorityMode function', () => {
-        // Use regex to be flexible with whitespace
-        expect(SITEGRAPH_HTML).toMatch(/function\s+setAuthorityMode\s*\(mode,\s*btn\)/);
-        expect(SITEGRAPH_HTML).toContain('n.authority = mode === \'pagerank\' ? n.pageRankAuthority : n.structuralAuthority');
-    });
-
-    it('should contain logic to calculate pageRankAuthority from pageRankScore', () => {
-        expect(SITEGRAPH_HTML).toContain('n.pageRankAuthority = n.pageRankScore / 100');
-        expect(SITEGRAPH_HTML).toContain('n.structuralAuthority = Math.log(1 + n.inLinks)');
-    });
-
-    it('should update details panel to show both metrics', () => {
-        expect(SITEGRAPH_HTML).toContain('id="d-auth-container"');
-        expect(SITEGRAPH_HTML).toContain('In-Degree: ${structVal}');
-        expect(SITEGRAPH_HTML).toContain('PR: <strong>${prVal}</strong>');
-    });
-});

package/tsconfig.json

@@ -1,11 +0,0 @@
-{
-    "extends": "../../tsconfig.json",
-    "compilerOptions": {
-        "outDir": "dist",
-        "rootDir": "src",
-        "declaration": true
-    },
-    "include": [
-        "src"
-    ]
-}