npm - @crawlith/core - Versions diffs - 0.1.1 → 0.1.2 - Mend

@crawlith/core 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (237) hide show

package/LICENSE +201 -0
package/README.md +70 -0
package/dist/analysis/analyze.d.ts +29 -8
package/dist/analysis/analyze.js +325 -221
package/dist/analysis/clustering.d.ts +23 -0
package/dist/analysis/clustering.js +206 -0
package/dist/analysis/content.d.ts +1 -1
package/dist/analysis/content.js +11 -5
package/dist/analysis/duplicate.d.ts +34 -0
package/dist/analysis/duplicate.js +305 -0
package/dist/analysis/heading.d.ts +116 -0
package/dist/analysis/heading.js +356 -0
package/dist/analysis/images.d.ts +1 -1
package/dist/analysis/images.js +6 -5
package/dist/analysis/links.d.ts +1 -1
package/dist/analysis/links.js +8 -8
package/dist/{scoring/orphanSeverity.d.ts → analysis/orphan.d.ts} +12 -23
package/dist/{scoring/orphanSeverity.js → analysis/orphan.js} +9 -3
package/dist/analysis/scoring.js +4 -1
package/dist/analysis/seo.d.ts +8 -4
package/dist/analysis/seo.js +41 -30
package/dist/analysis/soft404.d.ts +17 -0
package/dist/analysis/soft404.js +62 -0
package/dist/analysis/structuredData.d.ts +1 -1
package/dist/analysis/structuredData.js +5 -4
package/dist/application/index.d.ts +2 -0
package/dist/application/index.js +2 -0
package/dist/application/usecase.d.ts +3 -0
package/dist/application/usecase.js +1 -0
package/dist/application/usecases.d.ts +114 -0
package/dist/application/usecases.js +201 -0
package/dist/audit/index.js +1 -1
package/dist/audit/transport.d.ts +1 -1
package/dist/audit/transport.js +5 -4
package/dist/audit/types.d.ts +1 -0
package/dist/constants.d.ts +17 -0
package/dist/constants.js +23 -0
package/dist/core/scope/scopeManager.js +3 -0
package/dist/crawler/crawl.d.ts +2 -2
package/dist/crawler/crawler.d.ts +17 -5
package/dist/crawler/crawler.js +259 -94
package/dist/crawler/fetcher.d.ts +1 -1
package/dist/crawler/fetcher.js +6 -6
package/dist/crawler/metricsRunner.d.ts +21 -1
package/dist/crawler/metricsRunner.js +181 -60
package/dist/crawler/normalize.d.ts +41 -0
package/dist/crawler/normalize.js +119 -3
package/dist/crawler/parser.d.ts +1 -3
package/dist/crawler/parser.js +2 -49
package/dist/crawler/resolver.d.ts +11 -0
package/dist/crawler/resolver.js +67 -0
package/dist/crawler/sitemap.d.ts +4 -1
package/dist/crawler/sitemap.js +24 -18
package/dist/crawler/trap.d.ts +5 -1
package/dist/crawler/trap.js +23 -2
package/dist/db/CrawlithDB.d.ts +110 -0
package/dist/db/CrawlithDB.js +500 -0
package/dist/db/graphLoader.js +15 -32
package/dist/db/index.d.ts +9 -1
package/dist/db/index.js +39 -31
package/dist/db/migrations.d.ts +2 -0
package/dist/db/{schema.js → migrations.js} +90 -43
package/dist/db/pluginRegistry.d.ts +9 -0
package/dist/db/pluginRegistry.js +19 -0
package/dist/db/repositories/EdgeRepository.d.ts +5 -0
package/dist/db/repositories/EdgeRepository.js +7 -0
package/dist/db/repositories/MetricsRepository.d.ts +13 -8
package/dist/db/repositories/MetricsRepository.js +14 -6
package/dist/db/repositories/PageRepository.d.ts +5 -3
package/dist/db/repositories/PageRepository.js +68 -17
package/dist/db/repositories/SiteRepository.d.ts +6 -0
package/dist/db/repositories/SiteRepository.js +4 -0
package/dist/db/repositories/SnapshotRepository.d.ts +12 -5
package/dist/db/repositories/SnapshotRepository.js +48 -10
package/dist/db/reset.d.ts +9 -0
package/dist/db/reset.js +32 -0
package/dist/db/statements.d.ts +12 -0
package/dist/db/statements.js +40 -0
package/dist/diff/compare.d.ts +0 -5
package/dist/diff/compare.js +0 -12
package/dist/diff/service.d.ts +16 -0
package/dist/diff/service.js +41 -0
package/dist/domain/index.d.ts +4 -0
package/dist/domain/index.js +4 -0
package/dist/events.d.ts +8 -0
package/dist/graph/graph.d.ts +20 -42
package/dist/graph/graph.js +12 -16
package/dist/graph/hits.d.ts +23 -0
package/dist/graph/hits.js +111 -0
package/dist/graph/metrics.d.ts +0 -4
package/dist/graph/metrics.js +19 -15
package/dist/graph/pagerank.d.ts +17 -4
package/dist/graph/pagerank.js +126 -93
package/dist/index.d.ts +27 -9
package/dist/index.js +27 -9
package/dist/lock/lockManager.d.ts +1 -0
package/dist/lock/lockManager.js +15 -0
package/dist/plugin-system/plugin-cli.d.ts +10 -0
package/dist/plugin-system/plugin-cli.js +31 -0
package/dist/plugin-system/plugin-config.d.ts +16 -0
package/dist/plugin-system/plugin-config.js +36 -0
package/dist/plugin-system/plugin-loader.d.ts +17 -0
package/dist/plugin-system/plugin-loader.js +122 -0
package/dist/plugin-system/plugin-registry.d.ts +25 -0
package/dist/plugin-system/plugin-registry.js +167 -0
package/dist/plugin-system/plugin-types.d.ts +205 -0
package/dist/plugin-system/plugin-types.js +1 -0
package/dist/ports/index.d.ts +9 -0
package/dist/ports/index.js +1 -0
package/dist/report/export.d.ts +3 -0
package/dist/report/export.js +81 -0
package/dist/report/insight.d.ts +27 -0
package/dist/report/insight.js +103 -0
package/dist/scoring/health.d.ts +17 -11
package/dist/scoring/health.js +183 -140
package/dist/utils/chalk.d.ts +6 -0
package/dist/utils/chalk.js +41 -0
package/dist/utils/secureConfig.d.ts +23 -0
package/dist/utils/secureConfig.js +128 -0
package/package.json +10 -4
package/CHANGELOG.md +0 -13
package/dist/db/schema.d.ts +0 -2
package/dist/graph/cluster.d.ts +0 -6
package/dist/graph/cluster.js +0 -221
package/dist/graph/duplicate.d.ts +0 -10
package/dist/graph/duplicate.js +0 -302
package/dist/scoring/hits.d.ts +0 -10
package/dist/scoring/hits.js +0 -131
package/scripts/copy-assets.js +0 -37
package/src/analysis/analysis_list.html +0 -35
package/src/analysis/analysis_page.html +0 -123
package/src/analysis/analyze.ts +0 -505
package/src/analysis/content.ts +0 -62
package/src/analysis/images.ts +0 -28
package/src/analysis/links.ts +0 -41
package/src/analysis/scoring.ts +0 -66
package/src/analysis/seo.ts +0 -82
package/src/analysis/structuredData.ts +0 -62
package/src/analysis/templates.ts +0 -9
package/src/audit/dns.ts +0 -49
package/src/audit/headers.ts +0 -98
package/src/audit/index.ts +0 -66
package/src/audit/scoring.ts +0 -232
package/src/audit/transport.ts +0 -258
package/src/audit/types.ts +0 -102
package/src/core/network/proxyAdapter.ts +0 -21
package/src/core/network/rateLimiter.ts +0 -39
package/src/core/network/redirectController.ts +0 -47
package/src/core/network/responseLimiter.ts +0 -34
package/src/core/network/retryPolicy.ts +0 -57
package/src/core/scope/domainFilter.ts +0 -45
package/src/core/scope/scopeManager.ts +0 -52
package/src/core/scope/subdomainPolicy.ts +0 -39
package/src/core/security/ipGuard.ts +0 -171
package/src/crawler/crawl.ts +0 -9
package/src/crawler/crawler.ts +0 -601
package/src/crawler/extract.ts +0 -39
package/src/crawler/fetcher.ts +0 -251
package/src/crawler/metricsRunner.ts +0 -137
package/src/crawler/normalize.ts +0 -108
package/src/crawler/parser.ts +0 -190
package/src/crawler/sitemap.ts +0 -76
package/src/crawler/trap.ts +0 -96
package/src/db/graphLoader.ts +0 -135
package/src/db/index.ts +0 -75
package/src/db/repositories/EdgeRepository.ts +0 -43
package/src/db/repositories/MetricsRepository.ts +0 -63
package/src/db/repositories/PageRepository.ts +0 -228
package/src/db/repositories/SiteRepository.ts +0 -43
package/src/db/repositories/SnapshotRepository.ts +0 -99
package/src/db/schema.ts +0 -177
package/src/diff/compare.ts +0 -84
package/src/events.ts +0 -16
package/src/graph/cluster.ts +0 -246
package/src/graph/duplicate.ts +0 -350
package/src/graph/graph.ts +0 -192
package/src/graph/metrics.ts +0 -125
package/src/graph/pagerank.ts +0 -126
package/src/graph/simhash.ts +0 -76
package/src/index.ts +0 -33
package/src/lock/hashKey.ts +0 -51
package/src/lock/lockManager.ts +0 -132
package/src/lock/pidCheck.ts +0 -13
package/src/report/crawl.html +0 -879
package/src/report/crawlExport.ts +0 -58
package/src/report/crawl_template.ts +0 -9
package/src/report/html.ts +0 -27
package/src/scoring/health.ts +0 -241
package/src/scoring/hits.ts +0 -153
package/src/scoring/orphanSeverity.ts +0 -176
package/src/utils/version.ts +0 -18
package/tests/__snapshots__/orphanSeverity.test.ts.snap +0 -49
package/tests/analysis.unit.test.ts +0 -142
package/tests/analyze.integration.test.ts +0 -133
package/tests/analyze_markdown.test.ts +0 -98
package/tests/audit/audit.test.ts +0 -101
package/tests/audit/dns.test.ts +0 -31
package/tests/audit/headers.test.ts +0 -45
package/tests/audit/scoring.test.ts +0 -133
package/tests/audit/security.test.ts +0 -12
package/tests/audit/transport.test.ts +0 -111
package/tests/clustering.test.ts +0 -118
package/tests/clustering_risk.test.ts +0 -118
package/tests/crawler.test.ts +0 -364
package/tests/db/index.test.ts +0 -134
package/tests/db/repositories.test.ts +0 -115
package/tests/db.test.ts +0 -159
package/tests/db_repos.test.ts +0 -72
package/tests/diff.test.ts +0 -67
package/tests/duplicate.test.ts +0 -110
package/tests/extract.test.ts +0 -86
package/tests/fetcher.test.ts +0 -110
package/tests/fetcher_safety.test.ts +0 -91
package/tests/fixtures/analyze-crawl.json +0 -26
package/tests/graph/graph.test.ts +0 -100
package/tests/graphLoader.test.ts +0 -124
package/tests/hits.test.ts +0 -134
package/tests/html_report.test.ts +0 -59
package/tests/ipGuard.test.ts +0 -73
package/tests/lock/lockManager.test.ts +0 -198
package/tests/metrics.test.ts +0 -196
package/tests/normalize.test.ts +0 -88
package/tests/orphanSeverity.test.ts +0 -160
package/tests/pagerank.test.ts +0 -98
package/tests/parser.test.ts +0 -117
package/tests/proxy_safety.test.ts +0 -57
package/tests/redirect_safety.test.ts +0 -77
package/tests/renderAnalysisCsv.test.ts +0 -183
package/tests/safety.test.ts +0 -126
package/tests/scope.test.ts +0 -84
package/tests/scoring.test.ts +0 -60
package/tests/sitemap.test.ts +0 -100
package/tests/soft404.test.ts +0 -41
package/tests/ssrf_fix.test.ts +0 -69
package/tests/trap.test.ts +0 -39
package/tests/visualization_data.test.ts +0 -46
package/tsconfig.json +0 -11

package/tests/fetcher_safety.test.ts DELETED Viewed

@@ -1,91 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { Fetcher } from '../src/crawler/fetcher.js';
-import { request } from 'undici';
-vi.mock('undici', () => {
-    return {
-        request: vi.fn(),
-        Agent: class {
-            dispatch = vi.fn();
-        },
-        Dispatcher: class {}
-    };
-});
-describe('Fetcher Safety Integration', () => {
-    let fetcher: Fetcher;
-    beforeEach(() => {
-        vi.clearAllMocks();
-        fetcher = new Fetcher({ rate: 100 }); // High rate for tests
-    });
-    it('should block internal IPs', async () => {
-        const res = await fetcher.fetch('http://127.0.0.1');
-        expect(res.status).toBe('blocked_internal_ip');
-    });
-    it('should block internal IPs in redirects', async () => {
-        const mockRequest = vi.mocked(request);
-        // First request is fine, returns redirect
-        mockRequest.mockResolvedValueOnce({
-            statusCode: 301,
-            headers: { location: 'http://192.168.1.1' },
-            body: { dump: vi.fn(), text: vi.fn().mockResolvedValue('') }
-        } as any);
-        const res = await fetcher.fetch('http://example.com');
-        expect(res.status).toBe('blocked_internal_ip');
-        expect(res.redirectChain).toHaveLength(1); // Records the redirect that led to block
-        expect(res.redirectChain[0].target).toBe('http://192.168.1.1/');
-    });
-    it('should enforce max bytes', async () => {
-        const mockRequest = vi.mocked(request);
-        mockRequest.mockResolvedValueOnce({
-            statusCode: 200,
-            headers: {},
-            body: {
-                on: vi.fn((event, cb) => {
-                    if (event === 'data') {
-                        cb(Buffer.alloc(1000));
-                        cb(Buffer.alloc(1000));
-                    }
-                    return { on: vi.fn() };
-                }),
-                destroy: vi.fn(),
-                dump: vi.fn()
-            }
-        } as any);
-        const res = await fetcher.fetch('http://example.com', { maxBytes: 500 });
-        expect(res.status).toBe('oversized');
-    });
-    it('should retry on 500', async () => {
-        const mockRequest = vi.mocked(request);
-        mockRequest
-            .mockResolvedValueOnce({
-                statusCode: 500,
-                headers: {},
-                body: { dump: vi.fn().mockResolvedValue(undefined) }
-            } as any)
-            .mockResolvedValueOnce({
-                statusCode: 200,
-                headers: {},
-                body: {
-                    on: vi.fn((event, cb) => {
-                        if (event === 'data') cb(Buffer.from('ok'));
-                        if (event === 'end') cb();
-                    })
-                }
-            } as any);
-        const res = await fetcher.fetch('http://example.com');
-        expect(res.status).toBe(200);
-        expect(res.retries).toBe(1);
-    });
-});

package/tests/fixtures/analyze-crawl.json DELETED Viewed

@@ -1,26 +0,0 @@
-{
-  "pages": [
-    {
-      "url": "https://example.com/",
-      "status": 200,
-      "depth": 0,
-      "html": "<html><head><title>Example Home Page SEO Title For Strong Ranking Signals 12345</title><meta name='description' content='This is an intentionally long and descriptive meta description designed to fit ideal search snippet lengths with rich context for users and engines.'/></head><body><h1>Home</h1><nav><a href='/skip'>Nav</a></nav><p>Welcome to the homepage. This page contains meaningful content. Another sentence here.</p><img src='/a.jpg' alt='hero'><img src='/b.jpg'><a href='/about'>About</a><a href='https://external.com' rel='nofollow noopener'>External</a><script type='application/ld+json'>{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\"}</script></body></html>"
-    },
-    {
-      "url": "https://example.com/about",
-      "status": 200,
-      "depth": 1,
-      "html": "<html><head><title>Example Home Page SEO Title For Strong Ranking Signals 12345</title><meta name='description' content='short desc'/></head><body><h1>Example Home Page SEO Title For Strong Ranking Signals 12345</h1><h1>Second</h1><p>Duplicate body sentence. Duplicate body sentence.</p><img src='/c.jpg' alt=''><script type='application/ld+json'>not-json</script><a href='https://example.com/'>Home</a></body></html>"
-    },
-    {
-      "url": "https://example.com/empty",
-      "status": 200,
-      "depth": 2,
-      "html": ""
-    }
-  ],
-  "edges": [
-    { "source": "https://example.com/", "target": "https://example.com/about" },
-    { "source": "https://example.com/about", "target": "https://example.com/" }
-  ]
-}

package/tests/graph/graph.test.ts DELETED Viewed

@@ -1,100 +0,0 @@
-import { describe, it, expect, beforeEach } from 'vitest';
-import { Graph } from '../../src/graph/graph.js';
-describe('Graph', () => {
-  let graph: Graph;
-  beforeEach(() => {
-    graph = new Graph();
-  });
-  it('should add a new node', () => {
-    graph.addNode('http://example.com', 0, 200);
-    const node = graph.nodes.get('http://example.com');
-    expect(node).toBeDefined();
-    expect(node?.depth).toBe(0);
-    expect(node?.status).toBe(200);
-  });
-  it('should update existing node status if non-zero', () => {
-    graph.addNode('http://example.com', 0, 0);
-    graph.addNode('http://example.com', 1, 200); // Should update status, but not depth?
-    // Wait, addNode implementation:
-    // if (!existing) { ... } else { if (status !== 0) existing.status = status; }
-    const node = graph.nodes.get('http://example.com');
-    expect(node?.status).toBe(200);
-    expect(node?.depth).toBe(0); // Depth should not change
-  });
-  it('should add an edge', () => {
-    graph.addNode('http://a.com', 0);
-    graph.addNode('http://b.com', 1);
-    graph.addEdge('http://a.com', 'http://b.com', 0.5);
-    const edgeKey = Graph.getEdgeKey('http://a.com', 'http://b.com');
-    expect(graph.edges.has(edgeKey)).toBe(true);
-    expect(graph.edges.get(edgeKey)).toBe(0.5);
-    const source = graph.nodes.get('http://a.com');
-    const target = graph.nodes.get('http://b.com');
-    expect(source?.outLinks).toBe(1);
-    expect(target?.inLinks).toBe(1);
-  });
-  it('should update edge weight if new weight is higher', () => {
-    graph.addNode('http://a.com', 0);
-    graph.addNode('http://b.com', 1);
-    graph.addEdge('http://a.com', 'http://b.com', 0.5);
-    graph.addEdge('http://a.com', 'http://b.com', 0.8);
-    const edgeKey = Graph.getEdgeKey('http://a.com', 'http://b.com');
-    expect(graph.edges.get(edgeKey)).toBe(0.8);
-    // Should not increment link counts again
-    const source = graph.nodes.get('http://a.com');
-    expect(source?.outLinks).toBe(1);
-  });
-  it('should not update edge weight if new weight is lower', () => {
-    graph.addNode('http://a.com', 0);
-    graph.addNode('http://b.com', 1);
-    graph.addEdge('http://a.com', 'http://b.com', 0.8);
-    graph.addEdge('http://a.com', 'http://b.com', 0.5);
-    const edgeKey = Graph.getEdgeKey('http://a.com', 'http://b.com');
-    expect(graph.edges.get(edgeKey)).toBe(0.8);
-  });
-  it('should serialize to JSON and deserialize from JSON', () => {
-    graph.addNode('http://a.com', 0, 200);
-    graph.addNode('http://b.com', 1, 200);
-    graph.addEdge('http://a.com', 'http://b.com', 1.0);
-    graph.duplicateClusters = [{ id: '1', type: 'exact', size: 2, representative: 'http://a.com', severity: 'high' }];
-    graph.contentClusters = [{ id: 1, count: 2, primaryUrl: 'http://a.com', risk: 'high' }];
-    const json = graph.toJSON();
-    const newGraph = Graph.fromJSON(json);
-    expect(newGraph.nodes.size).toBe(2);
-    expect(newGraph.edges.size).toBe(1);
-    expect(newGraph.duplicateClusters).toHaveLength(1);
-    expect(newGraph.contentClusters).toHaveLength(1);
-    const nodeA = newGraph.nodes.get('http://a.com');
-    expect(nodeA?.status).toBe(200);
-    const edgeKey = Graph.getEdgeKey('http://a.com', 'http://b.com');
-    expect(newGraph.edges.get(edgeKey)).toBe(1.0);
-  });
-  it('should handle partial JSON in fromJSON', () => {
-    const json = {
-      nodes: [{ url: 'http://a.com', depth: 0, status: 200, inLinks: 0, outLinks: 0 }],
-      // missing edges, clusters
-    };
-    const newGraph = Graph.fromJSON(json);
-    expect(newGraph.nodes.size).toBe(1);
-    expect(newGraph.edges.size).toBe(0);
-  });
-});

package/tests/graphLoader.test.ts DELETED Viewed

@@ -1,124 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
-import { loadGraphFromSnapshot } from '../src/db/graphLoader.js';
-import { getDb, closeDb } from '../src/db/index.js';
-import { SiteRepository } from '../src/db/repositories/SiteRepository.js';
-import { SnapshotRepository } from '../src/db/repositories/SnapshotRepository.js';
-import { PageRepository } from '../src/db/repositories/PageRepository.js';
-import { MetricsRepository } from '../src/db/repositories/MetricsRepository.js';
-import { Database } from 'better-sqlite3';
-describe('GraphLoader', () => {
-    let db: Database;
-    beforeEach(() => {
-        process.env.NODE_ENV = 'test';
-        closeDb();
-        db = getDb();
-    });
-    afterEach(() => {
-        closeDb();
-    });
-    it('should load graph with metrics correctly', () => {
-        const siteRepo = new SiteRepository(db);
-        const snapshotRepo = new SnapshotRepository(db);
-        const pageRepo = new PageRepository(db);
-        const metricsRepo = new MetricsRepository(db);
-        const siteId = siteRepo.createSite('example.com');
-        const snapshotId = snapshotRepo.createSnapshot(siteId, 'full');
-        const url = 'http://example.com/page1';
-        // Create Page
-        pageRepo.upsertPage({
-            site_id: siteId,
-            normalized_url: url,
-            last_seen_snapshot_id: snapshotId,
-            http_status: 200,
-            depth: 0
-        });
-        const page = pageRepo.getPage(siteId, url)!;
-        // Insert Metrics
-        metricsRepo.insertMetrics({
-            snapshot_id: snapshotId,
-            page_id: page.id,
-            authority_score: 0.5,
-            hub_score: 0.2,
-            pagerank: 0.8,
-            pagerank_score: 80.0,
-            link_role: 'authority',
-            crawl_status: 'fetched',
-            word_count: 500,
-            thin_content_score: 10,
-            external_link_ratio: 0.1,
-            orphan_score: 5,
-            duplicate_cluster_id: null,
-            duplicate_type: null,
-            is_cluster_primary: 1
-        });
-        // Load Graph
-        const graph = loadGraphFromSnapshot(snapshotId);
-        const node = graph.nodes.get(url);
-        expect(node).toBeDefined();
-        expect(node?.authorityScore).toBe(0.5);
-        expect(node?.hubScore).toBe(0.2);
-        // Verify new fields
-        expect(node?.crawlStatus).toBe('fetched');
-        expect(node?.wordCount).toBe(500);
-        expect(node?.thinContentScore).toBe(10);
-        expect(node?.externalLinkRatio).toBe(0.1);
-        expect(node?.orphanScore).toBe(5);
-    });
-    it('should handle null metrics gracefully', () => {
-        const siteRepo = new SiteRepository(db);
-        const snapshotRepo = new SnapshotRepository(db);
-        const pageRepo = new PageRepository(db);
-        const metricsRepo = new MetricsRepository(db);
-        const siteId = siteRepo.createSite('example.com');
-        const snapshotId = snapshotRepo.createSnapshot(siteId, 'full');
-        const url = 'http://example.com/page2';
-        pageRepo.upsertPage({
-            site_id: siteId,
-            normalized_url: url,
-            last_seen_snapshot_id: snapshotId,
-            http_status: 200,
-            depth: 1
-        });
-        const page = pageRepo.getPage(siteId, url)!;
-        // Insert Metrics with nulls
-        metricsRepo.insertMetrics({
-            snapshot_id: snapshotId,
-            page_id: page.id,
-            authority_score: null,
-            hub_score: null,
-            pagerank: null,
-            pagerank_score: null,
-            link_role: null,
-            crawl_status: null,
-            word_count: null,
-            thin_content_score: null,
-            external_link_ratio: null,
-            orphan_score: null,
-            duplicate_cluster_id: null,
-            duplicate_type: null,
-            is_cluster_primary: 0
-        });
-        const graph = loadGraphFromSnapshot(snapshotId);
-        const node = graph.nodes.get(url);
-        expect(node).toBeDefined();
-        // Check undefined
-        expect(node?.crawlStatus).toBeUndefined();
-        expect(node?.wordCount).toBeUndefined();
-        expect(node?.thinContentScore).toBeUndefined();
-    });
-});

package/tests/hits.test.ts DELETED Viewed

@@ -1,134 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { Graph } from '../src/graph/graph.js';
-import { computeHITS } from '../src/scoring/hits.js';
-describe('HITS Scoring', () => {
-    it('should compute scores for a simple star topology', () => {
-        const graph = new Graph();
-        // Hub
-        graph.addNode('http://hub.com', 0, 200);
-        // Authorities
-        graph.addNode('http://auth1.com', 1, 200);
-        graph.addNode('http://auth2.com', 1, 200);
-        graph.addNode('http://auth3.com', 1, 200);
-        graph.addEdge('http://hub.com', 'http://auth1.com');
-        graph.addEdge('http://hub.com', 'http://auth2.com');
-        graph.addEdge('http://hub.com', 'http://auth3.com');
-        computeHITS(graph, { iterations: 10 });
-        const hub = graph.nodes.get('http://hub.com')!;
-        const auth1 = graph.nodes.get('http://auth1.com')!;
-        // In a star topology:
-        // Hub should have max hub score
-        // Authorities should have max authority scores
-        expect(hub.hubScore).toBeGreaterThan(0.9);
-        expect(hub.authorityScore).toBe(0); // No one links to hub
-        expect(auth1.authorityScore).toBeGreaterThan(0.5);
-        expect(auth1.hubScore).toBe(0); // Auth1 links to no one
-    });
-    it('should handle exclusion rules', () => {
-        const graph = new Graph();
-        graph.addNode('http://valid.com', 0, 200);
-        graph.addNode('http://noindex.com', 0, 200);
-        graph.updateNodeData('http://noindex.com', { noindex: true });
-        graph.addNode('http://redirect.com', 0, 200);
-        graph.updateNodeData('http://redirect.com', { redirectChain: ['http://target.com'] });
-        graph.addNode('http://external.com', 0, 200); // Eligibility check marks it as eligible if status is 200
-        // but typically external wouldn't have status 200 in the graph if we don't crawl them or they are marked as external.
-        // The current hits logic relies on: status === 200 && no redirectChain && !noindex
-        graph.addEdge('http://valid.com', 'http://noindex.com');
-        graph.addEdge('http://valid.com', 'http://redirect.com');
-        computeHITS(graph);
-        expect(graph.nodes.get('http://noindex.com')?.hubScore).toBeUndefined();
-        expect(graph.nodes.get('http://redirect.com')?.hubScore).toBeUndefined();
-        expect(graph.nodes.get('http://valid.com')?.hubScore).toBe(0); // Valid hub but its targets are ineligible
-    });
-    it('should respect edge weights', () => {
-        const graph = new Graph();
-        graph.addNode('http://hub.com', 0, 200);
-        graph.addNode('http://auth-high.com', 1, 200);
-        graph.addNode('http://auth-low.com', 1, 200);
-        graph.addEdge('http://hub.com', 'http://auth-high.com', 1.0);
-        graph.addEdge('http://hub.com', 'http://auth-low.com', 0.1);
-        computeHITS(graph, { iterations: 10 });
-        const authHigh = graph.nodes.get('http://auth-high.com')!;
-        const authLow = graph.nodes.get('http://auth-low.com')!;
-        expect(authHigh.authorityScore).toBeGreaterThan(authLow.authorityScore!);
-    });
-    it('should classify link roles correctly', () => {
-        const graph = new Graph();
-        for (let i = 0; i < 11; i++) {
-            graph.addNode(`http://node${i}.com`, 0, 200);
-        }
-        // AUTHORITY: node1 (linked by 0,2,3... no outlinks)
-        graph.addEdge('http://node0.com', 'http://node1.com');
-        graph.addEdge('http://node2.com', 'http://node1.com');
-        graph.addEdge('http://node3.com', 'http://node1.com');
-        graph.addEdge('http://node4.com', 'http://node1.com');
-        // HUB: node4 (links to 1,5,6,7... few inlinks)
-        graph.addEdge('http://node4.com', 'http://node5.com');
-        graph.addEdge('http://node4.com', 'http://node6.com');
-        graph.addEdge('http://node4.com', 'http://node7.com');
-        // POWER: node2 (linked by 0, power is often recursive... link to authority and be linked by hub)
-        graph.addEdge('http://node0.com', 'http://node2.com');
-        graph.addEdge('http://node2.com', 'http://node1.com');
-        graph.addEdge('http://node2.com', 'http://node5.com');
-        // PERIPHERAL: node10 (no links)
-        // Some filler nodes to push medians down
-        graph.addEdge('http://node8.com', 'http://node9.com');
-        computeHITS(graph, { iterations: 20 });
-        const roles = graph.getNodes().map(n => n.linkRole).filter(Boolean);
-        expect(roles).toContain('authority');
-        expect(roles).toContain('hub');
-        expect(roles).toContain('power');
-        expect(roles).toContain('peripheral');
-    });
-    it('should handle large synthetic graphs (Performance Test)', () => {
-        const graph = new Graph();
-        const nodeCount = 5000;
-        // Create 5000 nodes
-        for (let i = 0; i < nodeCount; i++) {
-            graph.addNode(`http://page${i}.com`, 1, 200);
-        }
-        // Create random edges (avg 10 per node)
-        for (let i = 0; i < nodeCount; i++) {
-            for (let j = 0; j < 10; j++) {
-                const target = Math.floor(Math.random() * nodeCount);
-                if (i !== target) {
-                    graph.addEdge(`http://page${i}.com`, `http://page${target}.com`);
-                }
-            }
-        }
-        const start = Date.now();
-        computeHITS(graph, { iterations: 20 });
-        const duration = Date.now() - start;
-        console.log(`HITS on 5000 nodes took ${duration}ms`);
-        expect(duration).toBeLessThan(2000); // Should be very fast, but allow buffer for CI environments
-        expect(graph.nodes.get('http://page0.com')?.hubScore).toBeDefined();
-    });
-});

package/tests/html_report.test.ts DELETED Viewed

@@ -1,59 +0,0 @@
-import { describe, expect, test } from 'vitest';
-import { renderAnalysisHtml, AnalysisResult, PageAnalysis } from '../src/analysis/analyze.js';
-const mockPage: PageAnalysis = {
-  url: 'https://example.com',
-  status: 200,
-  seoScore: 85,
-  thinScore: 10,
-  title: { value: 'Example Title', length: 13, status: 'ok' },
-  metaDescription: { value: 'Example Desc', length: 12, status: 'ok' },
-  h1: { count: 1, status: 'ok', matchesTitle: true },
-  content: { wordCount: 500, uniqueSentenceCount: 50, textHtmlRatio: 0.6 },
-  images: { totalImages: 2, missingAlt: 0, emptyAlt: 0 },
-  links: { internalLinks: 5, externalLinks: 2, nofollowCount: 0, externalRatio: 0.2 },
-  structuredData: { present: true, valid: true, types: ['Article'] },
-  meta: { canonical: 'https://example.com', noindex: false, nofollow: false }
-};
-const mockResult: AnalysisResult = {
-  site_summary: {
-    pages_analyzed: 1,
-    avg_seo_score: 85,
-    thin_pages: 0,
-    duplicate_titles: 0,
-    site_score: 90
-  },
-  site_scores: { overallScore: 90, seoHealthScore: 85 },
-  pages: [mockPage],
-  active_modules: { seo: true, content: true, accessibility: true }
-};
-describe('HTML Report Generation', () => {
-  test('generates single page report correctly', () => {
-    // If pages length is 1, it renders single page report
-    const html = renderAnalysisHtml(mockResult);
-    expect(html).toContain('<!DOCTYPE html>');
-    expect(html).toContain('Analysis for https://example.com');
-    expect(html).toContain('Example Title');
-    expect(html).toContain('Example Desc');
-    expect(html).toContain('500 words');
-    expect(html).toContain('<span class="status-ok">Valid</span>');
-  });
-  test('generates list report correctly', () => {
-    // Modify result to have 2 pages to trigger list view
-    const listResult: AnalysisResult = {
-      ...mockResult,
-      pages: [mockPage, { ...mockPage, url: 'https://example.com/2' }]
-    };
-    const html = renderAnalysisHtml(listResult);
-    expect(html).toContain('<!DOCTYPE html>');
-    expect(html).toContain('Crawlith Analysis Report');
-    expect(html).toContain('Pages: 1'); // site_summary.pages_analyzed is 1 in mockResult
-    expect(html).toContain('https://example.com');
-    expect(html).toContain('https://example.com/2');
-    expect(html).toContain('<td>85</td>'); // seoScore
-  });
-});

package/tests/ipGuard.test.ts DELETED Viewed

@@ -1,73 +0,0 @@
-import { describe, it, expect, vi } from 'vitest';
-import { IPGuard } from '../src/core/security/ipGuard.js';
-import * as dns from 'dns';
-vi.mock('dns', () => ({
-    lookup: vi.fn(),
-    resolve4: vi.fn(),
-    resolve6: vi.fn(),
-}));
-describe('IPGuard Secure Lookup', () => {
-    it('should resolve safe IPs', () => {
-        const lookupMock = vi.mocked(dns.lookup);
-        // Mock successful resolution
-        lookupMock.mockImplementation((hostname, options, callback) => {
-            callback(null, '8.8.8.8', 4);
-        });
-        const callback = vi.fn();
-        IPGuard.secureLookup('google.com', {}, callback);
-        expect(callback).toHaveBeenCalledWith(null, '8.8.8.8', 4);
-    });
-    it('should block internal IPs', () => {
-        const lookupMock = vi.mocked(dns.lookup);
-        // Mock internal IP resolution
-        lookupMock.mockImplementation((hostname, options, callback) => {
-            callback(null, '127.0.0.1', 4);
-        });
-        const callback = vi.fn();
-        IPGuard.secureLookup('localhost', {}, callback);
-        expect(callback).toHaveBeenCalledWith(expect.any(Error), '127.0.0.1', 4);
-        const error = callback.mock.calls[0][0];
-        expect(error.message).toContain('Blocked internal IP');
-        expect(error.code).toBe('EBLOCKED');
-    });
-    it('should handle array of IPs (IPv4)', () => {
-        const lookupMock = vi.mocked(dns.lookup);
-        // Mock array resolution
-        lookupMock.mockImplementation((hostname, options, callback) => {
-             // Mocking address array structure
-             const addresses = [
-                 { address: '1.1.1.1', family: 4 },
-                 { address: '127.0.0.1', family: 4 }
-             ];
-             callback(null, addresses as any, 4);
-        });
-        const callback = vi.fn();
-        IPGuard.secureLookup('mixed.com', { all: true } as any, callback);
-        expect(callback).toHaveBeenCalledWith(expect.any(Error), expect.anything(), 4);
-        const error = callback.mock.calls[0][0];
-        expect(error.message).toContain('Blocked internal IP');
-    });
-    it('should pass through DNS errors', () => {
-        const lookupMock = vi.mocked(dns.lookup);
-        const dnsError = new Error('ENOTFOUND');
-        lookupMock.mockImplementation((hostname, options, callback) => {
-            callback(dnsError as any, undefined as any, 0);
-        });
-        const callback = vi.fn();
-        IPGuard.secureLookup('invalid.domain', {}, callback);
-        expect(callback).toHaveBeenCalledWith(dnsError, undefined, 0);
-    });
-});