@crawlith/core 0.1.1 → 0.1.2

package/tests/scope.test.ts

@@ -1,84 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { DomainFilter } from '../src/core/scope/domainFilter.js';
-import { SubdomainPolicy } from '../src/core/scope/subdomainPolicy.js';
-import { ScopeManager } from '../src/core/scope/scopeManager.js';
-
-describe('DomainFilter', () => {
-    it('should normalize hostnames', () => {
-        const filter = new DomainFilter(['EXAMPLE.COM.'], ['DENY.COM.']);
-        expect(filter.isAllowed('example.com')).toBe(true);
-        expect(filter.isAllowed('deny.com')).toBe(false);
-    });
-
-    it('should respect precedence (deny wins)', () => {
-        const filter = new DomainFilter(['example.com'], ['example.com']);
-        expect(filter.isAllowed('example.com')).toBe(false);
-    });
-
-    it('should handle punycode', () => {
-        // xn--80ak6aa92e.com is punycode for пример.com
-        const filter = new DomainFilter(['xn--80ak6aa92e.com']);
-        expect(filter.isAllowed('XN--80AK6AA92E.COM')).toBe(true);
-    });
-
-    it('should block if not in allow list (when list not empty)', () => {
-        const filter = new DomainFilter(['allowed.com']);
-        expect(filter.isAllowed('other.com')).toBe(false);
-    });
-
-    it('should fallback to raw string on invalid hostname', () => {
-        // '[' and 'http://denied-invalid-[' causes new URL() to throw
-        const filter = new DomainFilter(['['], ['denied-invalid-[']);
-        expect(filter.isAllowed('[')).toBe(true);
-        expect(filter.isAllowed('denied-invalid-[')).toBe(false);
-    });
-});
-
-describe('SubdomainPolicy', () => {
-    it('should enforce exact match by default', () => {
-        const policy = new SubdomainPolicy('https://example.com');
-        expect(policy.isAllowed('example.com')).toBe(true);
-        expect(policy.isAllowed('sub.example.com')).toBe(false);
-    });
-
-    it('should allow valid subdomains when enabled', () => {
-        const policy = new SubdomainPolicy('https://example.com', true);
-        expect(policy.isAllowed('example.com')).toBe(true);
-        expect(policy.isAllowed('sub.example.com')).toBe(true);
-        expect(policy.isAllowed('deep.sub.example.com')).toBe(true);
-    });
-
-    it('should reject malicious suffix matches', () => {
-        const policy = new SubdomainPolicy('https://example.com', true);
-        expect(policy.isAllowed('evil-example.com')).toBe(false);
-        expect(policy.isAllowed('example.com.evil.com')).toBe(false);
-    });
-});
-
-describe('ScopeManager', () => {
-    it('should compose policies correctly', () => {
-        const manager = new ScopeManager({
-            rootUrl: 'https://example.com',
-            allowedDomains: ['example.com', 'sub.example.com', 'other.com'],
-            deniedDomains: ['bad.example.com'],
-            includeSubdomains: true
-        });
-
-        expect(manager.isUrlEligible('https://example.com/')).toBe('allowed');
-        expect(manager.isUrlEligible('https://sub.example.com/')).toBe('allowed');
-        expect(manager.isUrlEligible('https://bad.example.com/')).toBe('blocked_by_domain_filter');
-        expect(manager.isUrlEligible('https://other.com/')).toBe('allowed');
-        expect(manager.isUrlEligible('https://google.com/')).toBe('blocked_by_domain_filter');
-    });
-
-    it('should handle trailing dots in hostnames', () => {
-        const manager = new ScopeManager({
-            rootUrl: 'https://example.com',
-            allowedDomains: ['example.com.'],
-            includeSubdomains: false
-        });
-
-        expect(manager.isUrlEligible('https://example.com./')).toBe('allowed');
-        expect(manager.isUrlEligible('https://example.com/')).toBe('allowed');
-    });
-});

package/tests/scoring.test.ts

@@ -1,60 +0,0 @@
-import { expect, test } from 'vitest';
-import { scorePageSeo } from '../src/analysis/scoring.js';
-import { PageAnalysis } from '../src/analysis/analyze.js';
-
-const basePage: PageAnalysis = {
-  url: 'https://example.com',
-  status: 200,
-  title: { value: 'x'.repeat(55), length: 55, status: 'ok' },
-  metaDescription: { value: 'x'.repeat(150), length: 150, status: 'ok' },
-  h1: { count: 1, status: 'ok', matchesTitle: false },
-  content: { wordCount: 700, textHtmlRatio: 0.3, uniqueSentenceCount: 8 },
-  thinScore: 0,
-  images: { totalImages: 2, missingAlt: 0, emptyAlt: 0 },
-  links: { internalLinks: 5, externalLinks: 2, nofollowCount: 1, externalRatio: 2 / 7 },
-  structuredData: { present: true, valid: true, types: ['Article'] },
-  seoScore: 0,
-  meta: { noindex: false, nofollow: false }
-};
-
-test('page score stays in 0-100', () => {
-  expect(scorePageSeo(basePage)).toBeGreaterThanOrEqual(0);
-  expect(scorePageSeo(basePage)).toBeLessThanOrEqual(100);
-
-  const badPage: PageAnalysis = {
-    ...basePage,
-    title: { value: null, length: 0, status: 'missing' },
-    metaDescription: { value: null, length: 0, status: 'missing' },
-    h1: { count: 0, status: 'critical', matchesTitle: false },
-    content: { wordCount: 0, textHtmlRatio: 0, uniqueSentenceCount: 0 },
-    thinScore: 100,
-    images: { totalImages: 2, missingAlt: 2, emptyAlt: 0 },
-    structuredData: { present: false, valid: false, types: [] },
-    links: { internalLinks: 0, externalLinks: 9, nofollowCount: 9, externalRatio: 1 }
-  };
-  expect(scorePageSeo(badPage)).toBeLessThan(50);
-});
-
-// test('aggregate site score includes existing metrics signals', () => {
-//   const score = aggregateSiteScore({
-//     totalPages: 2,
-//     totalEdges: 1,
-//     orphanPages: ['https://example.com/x'],
-//     nearOrphans: [],
-//     deepPages: [],
-//     topAuthorityPages: [{ url: 'a', authority: 1 }],
-//     averageOutDegree: 1,
-//     maxDepthFound: 1,
-//     crawlEfficiencyScore: 0.8,
-//     averageDepth: 1,
-//     structuralEntropy: 2,
-//     limitReached: false
-//   }, [
-//     { ...basePage, seoScore: 70 },
-//     { ...basePage, seoScore: 90, url: 'https://example.com/2' }
-//   ]);
-
-//   expect(score.seoHealthScore).toBe(80);
-//   expect(score.overallScore).toBeGreaterThan(0);
-//   expect(score.overallScore).toBeLessThanOrEqual(100);
-// });

package/tests/sitemap.test.ts

@@ -1,100 +0,0 @@
-import { test, expect, beforeEach, vi } from 'vitest';
-import { Sitemap } from '../src/crawler/sitemap.js';
-import { MockAgent, setGlobalDispatcher } from 'undici';
-import { EngineContext } from '../src/events.js';
-
-let mockAgent: MockAgent;
-
-beforeEach(() => {
-  mockAgent = new MockAgent();
-  mockAgent.disableNetConnect();
-  setGlobalDispatcher(mockAgent);
-});
-
-test('fetches and parses simple sitemap', async () => {
-  const client = mockAgent.get('https://example.com');
-  client.intercept({
-    path: '/sitemap.xml',
-    method: 'GET'
-  }).reply(200, `
-    <?xml version="1.0" encoding="UTF-8"?>
-    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-      <url>
-        <loc>https://example.com/page1</loc>
-      </url>
-      <url>
-        <loc>https://example.com/page2</loc>
-      </url>
-    </urlset>
-  `);
-
-  const sitemap = new Sitemap();
-  const urls = await sitemap.fetch('https://example.com/sitemap.xml');
-  expect(urls).toContain('https://example.com/page1');
-  expect(urls).toContain('https://example.com/page2');
-  expect(urls.length).toBe(2);
-});
-
-test('handles sitemap index recursively', async () => {
-  const client = mockAgent.get('https://example.com');
-
-  // Index
-  client.intercept({
-    path: '/sitemap-index.xml',
-    method: 'GET'
-  }).reply(200, `
-    <?xml version="1.0" encoding="UTF-8"?>
-    <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-      <sitemap>
-        <loc>https://example.com/sitemap1.xml</loc>
-      </sitemap>
-    </sitemapindex>
-  `);
-
-  // Child sitemap
-  client.intercept({
-    path: '/sitemap1.xml',
-    method: 'GET'
-  }).reply(200, `
-    <?xml version="1.0" encoding="UTF-8"?>
-    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-      <url>
-        <loc>https://example.com/page3</loc>
-      </url>
-    </urlset>
-  `);
-
-  const sitemap = new Sitemap();
-  const urls = await sitemap.fetch('https://example.com/sitemap-index.xml');
-  expect(urls).toContain('https://example.com/page3');
-  expect(urls.length).toBe(1);
-});
-
-test('handles invalid xml gracefully', async () => {
-  const client = mockAgent.get('https://example.com');
-  client.intercept({ path: '/bad.xml', method: 'GET' }).reply(200, 'Not XML');
-
-  const sitemap = new Sitemap();
-  const urls = await sitemap.fetch('https://example.com/bad.xml');
-  expect(urls.length).toBe(0);
-});
-
-test('handles fetch errors gracefully', async () => {
-  const client = mockAgent.get('https://example.com');
-  client.intercept({ path: '/error.xml', method: 'GET' }).reply(500, 'Error');
-
-  const sitemap = new Sitemap();
-  const urls = await sitemap.fetch('https://example.com/error.xml');
-  expect(urls.length).toBe(0);
-});
-
-test('emits warning on fetch error', async () => {
-  const client = mockAgent.get('https://example.com');
-  client.intercept({ path: '/error.xml', method: 'GET' }).replyWithError(new Error('Network error'));
-
-  const mockContext: EngineContext = { emit: vi.fn() };
-  const sitemap = new Sitemap(mockContext);
-  await sitemap.fetch('https://example.com/error.xml');
-
-  expect(mockContext.emit).toHaveBeenCalledWith(expect.objectContaining({ type: 'warn' }));
-});

package/tests/soft404.test.ts

@@ -1,41 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { Parser } from '../src/crawler/parser.js';
-
-describe('Soft 404 Detection', () => {
-    const parser = new Parser();
-    const baseUrl = 'https://example.com';
-
-    it('should detect soft 404 by title pattern', () => {
-        const html = '<html><head><title>Page Not Found</title></head><body>Welcome to the site</body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        expect(result.soft404Score).toBeGreaterThan(0.3);
-        expect(result.soft404Signals).toContain('title_pattern_not_found');
-    });
-
-    it('should detect soft 404 by H1 pattern', () => {
-        const html = '<html><body><h1>404 Error</h1></body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        expect(result.soft404Score).toBeGreaterThan(0.2);
-        expect(result.soft404Signals).toContain('h1_pattern_404');
-    });
-
-    it('should detect soft 404 by very low word count', () => {
-        const html = '<html><body>Short text</body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        expect(result.soft404Score).toBeGreaterThan(0.2);
-        expect(result.soft404Signals).toContain('very_low_word_count');
-    });
-
-    it('should detect soft 404 by lack of outbound links', () => {
-        const html = '<html><body>A page with some text but no links.</body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        expect(result.soft404Signals).toContain('no_outbound_links');
-    });
-
-    it('should combine multiple signals for high score', () => {
-        const html = '<html><head><title>Error</title></head><body><h1>Not Found</h1><p>The requested page was not found.</p></body></html>';
-        const result = parser.parse(html, baseUrl, 200);
-        // title (0.4) + h1 (0.3) + body phrase (0.2) + low word count (0.3) = 1.2 -> capped at 1.0
-        expect(result.soft404Score).toBe(1.0);
-    });
-});

package/tests/ssrf_fix.test.ts

@@ -1,69 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { Fetcher } from '../src/crawler/fetcher.js';
-import { request } from 'undici';
-import { IPGuard } from '../src/core/security/ipGuard.js';
-
-// Mock undici request to fail with EBLOCKED
-vi.mock('undici', () => {
-    return {
-        request: vi.fn(),
-        Agent: class {
-            dispatch = vi.fn();
-        },
-        Dispatcher: class {}
-    };
-});
-
-// Mock IPGuard.validateHost to pass
-vi.mock('../src/core/security/ipGuard.js', async () => {
-    const original = await vi.importActual('../src/core/security/ipGuard.js');
-    return {
-        ...original as any,
-        IPGuard: {
-            ...original.IPGuard,
-            validateHost: vi.fn().mockResolvedValue(true), // Pass step 1
-            getSecureDispatcher: vi.fn()
-        }
-    };
-});
-
-describe('SSRF Fix Reproduction', () => {
-    let fetcher: Fetcher;
-
-    beforeEach(() => {
-        vi.clearAllMocks();
-        // Setup default mock return for dispatcher
-        vi.mocked(IPGuard.getSecureDispatcher).mockReturnValue({} as any);
-        fetcher = new Fetcher({ rate: 100 });
-    });
-
-    it('should return blocked_internal_ip when secureDispatcher blocks', async () => {
-        const mockRequest = vi.mocked(request);
-        const mockGetSecureDispatcher = vi.mocked(IPGuard.getSecureDispatcher);
-        const mockDispatcher = { dispatch: vi.fn() } as any;
-        mockGetSecureDispatcher.mockReturnValue(mockDispatcher);
-
-        // Re-initialize fetcher so it calls getSecureDispatcher and gets our specific mock
-        fetcher = new Fetcher({ rate: 100 });
-
-        // Simulate secureDispatcher blocking via undici request throwing EBLOCKED
-        const blockedError = new Error('Blocked internal IP: 127.0.0.1');
-        (blockedError as any).code = 'EBLOCKED';
-
-        mockRequest.mockRejectedValueOnce(blockedError);
-
-        const res = await fetcher.fetch('http://example.com');
-
-        // Now we expect correct handling
-        expect(res.status).toBe('blocked_internal_ip');
-
-        // Verify that the secure dispatcher was indeed used
-        expect(mockGetSecureDispatcher).toHaveBeenCalled();
-        expect(mockRequest).toHaveBeenCalledWith(
-            expect.stringContaining('http://example.com'),
-            expect.objectContaining({
-                dispatcher: mockDispatcher
-            })
-        );
-    });
-});

package/tests/trap.test.ts

@@ -1,39 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { TrapDetector } from '../src/crawler/trap.js';
-
-describe('TrapDetector', () => {
-    const detector = new TrapDetector();
-
-    it('should detect session ID traps', () => {
-        const result = detector.checkTrap('https://example.com/page?sid=12345', 1);
-        expect(result.risk).toBeGreaterThan(0.8);
-        expect(result.type).toBe('session_trap');
-    });
-
-    it('should detect calendar patterns', () => {
-        const result = detector.checkTrap('https://example.com/archive/2023/12/01/', 1);
-        expect(result.risk).toBeGreaterThan(0.6);
-        expect(result.type).toBe('calendar_trap');
-    });
-
-    it('should detect pagination loops', () => {
-        // Simulate many pages
-        for (let i = 1; i <= 60; i++) {
-            detector.checkTrap(`https://example.com/blog?page=${i}`, 1);
-        }
-        const result = detector.checkTrap('https://example.com/blog?page=61', 1);
-        expect(result.risk).toBeGreaterThan(0.8);
-        expect(result.type).toBe('pagination_loop');
-    });
-
-    it('should detect faceted navigation / parameter explosion', () => {
-        detector.reset();
-        const basePath = 'https://example.com/products';
-        for (let i = 1; i <= 35; i++) {
-            detector.checkTrap(`${basePath}?color=red&size=${i}`, 1);
-        }
-        const result = detector.checkTrap(`${basePath}?color=blue&size=large`, 1);
-        expect(result.risk).toBeGreaterThan(0.9);
-        expect(result.type).toBe('faceted_navigation');
-    });
-});

package/tests/visualization_data.test.ts

@@ -1,46 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { Crawl_HTML } from '../src/report/crawl_template.js';
-import { Graph } from '../src/graph/graph.js';
-import { computePageRank } from '../src/graph/pagerank.js';
-
-describe('Visualization Data & Template', () => {
-    it('should include pageRankScore in graph JSON output after PageRank computation', () => {
-        const graph = new Graph();
-        graph.addNode('https://a.com', 0, 200);
-        graph.addNode('https://b.com', 1, 200);
-        graph.addEdge('https://a.com', 'https://b.com');
-
-        computePageRank(graph);
-
-        const json = graph.toJSON();
-        const nodeA = json.nodes.find(n => n.url === 'https://a.com');
-        const nodeB = json.nodes.find(n => n.url === 'https://b.com');
-
-        expect(nodeA).toBeDefined();
-        expect(nodeB).toBeDefined();
-        expect(typeof nodeA?.pageRankScore).toBe('number');
-        expect(typeof nodeB?.pageRankScore).toBe('number');
-    });
-
-    it('should contain UI toggle buttons for Authority Mode', () => {
-        expect(Crawl_HTML).toContain('id="btn-auth-pagerank"');
-        expect(Crawl_HTML).toContain('id="btn-auth-structural"');
-    });
-
-    it('should contain setAuthorityMode function', () => {
-        // Use regex to be flexible with whitespace
-        expect(Crawl_HTML).toMatch(/function\s+setAuthorityMode\s*\(mode,\s*btn\)/);
-        expect(Crawl_HTML).toContain('n.authority = mode === \'pagerank\' ? n.pageRankAuthority : n.structuralAuthority');
-    });
-
-    it('should contain logic to calculate pageRankAuthority from pageRankScore', () => {
-        expect(Crawl_HTML).toContain('n.pageRankAuthority = n.pageRankScore / 100');
-        expect(Crawl_HTML).toContain('n.structuralAuthority = Math.log(1 + n.inLinks)');
-    });
-
-    it('should update details panel to show both metrics', () => {
-        expect(Crawl_HTML).toContain('id="d-auth-container"');
-        expect(Crawl_HTML).toContain('In-Degree: ${structVal}');
-        expect(Crawl_HTML).toContain('PR: <strong>${prVal}</strong>');
-    });
-});

package/tsconfig.json

@@ -1,11 +0,0 @@
-{
-    "extends": "../../tsconfig.json",
-    "compilerOptions": {
-        "outDir": "dist",
-        "rootDir": "src",
-        "declaration": true
-    },
-    "include": [
-        "src"
-    ]
-}