npm - @crawlith/core - Versions diffs - 0.1.0 → 0.1.2 - Mend

@crawlith/core 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (238) hide show

package/LICENSE +201 -0
package/README.md +70 -0
package/dist/analysis/analysis_list.html +35 -0
package/dist/analysis/analysis_page.html +123 -0
package/dist/analysis/analyze.d.ts +40 -5
package/dist/analysis/analyze.js +395 -347
package/dist/analysis/clustering.d.ts +23 -0
package/dist/analysis/clustering.js +206 -0
package/dist/analysis/content.d.ts +1 -1
package/dist/analysis/content.js +11 -5
package/dist/analysis/duplicate.d.ts +34 -0
package/dist/analysis/duplicate.js +305 -0
package/dist/analysis/heading.d.ts +116 -0
package/dist/analysis/heading.js +356 -0
package/dist/analysis/images.d.ts +1 -1
package/dist/analysis/images.js +6 -5
package/dist/analysis/links.d.ts +1 -1
package/dist/analysis/links.js +8 -8
package/dist/{scoring/orphanSeverity.d.ts → analysis/orphan.d.ts} +12 -23
package/dist/{scoring/orphanSeverity.js → analysis/orphan.js} +9 -3
package/dist/analysis/scoring.js +11 -2
package/dist/analysis/seo.d.ts +8 -4
package/dist/analysis/seo.js +41 -30
package/dist/analysis/soft404.d.ts +17 -0
package/dist/analysis/soft404.js +62 -0
package/dist/analysis/structuredData.d.ts +1 -1
package/dist/analysis/structuredData.js +5 -4
package/dist/analysis/templates.d.ts +2 -0
package/dist/analysis/templates.js +7 -0
package/dist/application/index.d.ts +2 -0
package/dist/application/index.js +2 -0
package/dist/application/usecase.d.ts +3 -0
package/dist/application/usecase.js +1 -0
package/dist/application/usecases.d.ts +114 -0
package/dist/application/usecases.js +201 -0
package/dist/audit/index.js +1 -1
package/dist/audit/transport.d.ts +1 -1
package/dist/audit/transport.js +5 -4
package/dist/audit/types.d.ts +1 -0
package/dist/constants.d.ts +17 -0
package/dist/constants.js +23 -0
package/dist/core/scope/scopeManager.js +3 -0
package/dist/core/security/ipGuard.d.ts +11 -0
package/dist/core/security/ipGuard.js +71 -3
package/dist/crawler/crawl.d.ts +4 -22
package/dist/crawler/crawl.js +4 -335
package/dist/crawler/crawler.d.ts +87 -0
package/dist/crawler/crawler.js +683 -0
package/dist/crawler/extract.d.ts +4 -1
package/dist/crawler/extract.js +7 -2
package/dist/crawler/fetcher.d.ts +2 -1
package/dist/crawler/fetcher.js +26 -11
package/dist/crawler/metricsRunner.d.ts +23 -1
package/dist/crawler/metricsRunner.js +202 -72
package/dist/crawler/normalize.d.ts +41 -0
package/dist/crawler/normalize.js +119 -3
package/dist/crawler/parser.d.ts +1 -3
package/dist/crawler/parser.js +2 -49
package/dist/crawler/resolver.d.ts +11 -0
package/dist/crawler/resolver.js +67 -0
package/dist/crawler/sitemap.d.ts +6 -0
package/dist/crawler/sitemap.js +27 -17
package/dist/crawler/trap.d.ts +5 -1
package/dist/crawler/trap.js +23 -2
package/dist/db/CrawlithDB.d.ts +110 -0
package/dist/db/CrawlithDB.js +500 -0
package/dist/db/graphLoader.js +42 -30
package/dist/db/index.d.ts +11 -0
package/dist/db/index.js +41 -29
package/dist/db/migrations.d.ts +2 -0
package/dist/db/{schema.js → migrations.js} +90 -43
package/dist/db/pluginRegistry.d.ts +9 -0
package/dist/db/pluginRegistry.js +19 -0
package/dist/db/repositories/EdgeRepository.d.ts +13 -0
package/dist/db/repositories/EdgeRepository.js +20 -0
package/dist/db/repositories/MetricsRepository.d.ts +16 -8
package/dist/db/repositories/MetricsRepository.js +28 -7
package/dist/db/repositories/PageRepository.d.ts +15 -2
package/dist/db/repositories/PageRepository.js +169 -25
package/dist/db/repositories/SiteRepository.d.ts +9 -0
package/dist/db/repositories/SiteRepository.js +13 -0
package/dist/db/repositories/SnapshotRepository.d.ts +14 -5
package/dist/db/repositories/SnapshotRepository.js +64 -5
package/dist/db/reset.d.ts +9 -0
package/dist/db/reset.js +32 -0
package/dist/db/statements.d.ts +12 -0
package/dist/db/statements.js +40 -0
package/dist/diff/compare.d.ts +0 -5
package/dist/diff/compare.js +0 -12
package/dist/diff/service.d.ts +16 -0
package/dist/diff/service.js +41 -0
package/dist/domain/index.d.ts +4 -0
package/dist/domain/index.js +4 -0
package/dist/events.d.ts +56 -0
package/dist/events.js +1 -0
package/dist/graph/graph.d.ts +36 -42
package/dist/graph/graph.js +26 -17
package/dist/graph/hits.d.ts +23 -0
package/dist/graph/hits.js +111 -0
package/dist/graph/metrics.d.ts +0 -4
package/dist/graph/metrics.js +25 -9
package/dist/graph/pagerank.d.ts +17 -4
package/dist/graph/pagerank.js +126 -91
package/dist/graph/simhash.d.ts +6 -0
package/dist/graph/simhash.js +14 -0
package/dist/index.d.ts +29 -8
package/dist/index.js +29 -8
package/dist/lock/hashKey.js +1 -1
package/dist/lock/lockManager.d.ts +5 -1
package/dist/lock/lockManager.js +38 -13
package/dist/plugin-system/plugin-cli.d.ts +10 -0
package/dist/plugin-system/plugin-cli.js +31 -0
package/dist/plugin-system/plugin-config.d.ts +16 -0
package/dist/plugin-system/plugin-config.js +36 -0
package/dist/plugin-system/plugin-loader.d.ts +17 -0
package/dist/plugin-system/plugin-loader.js +122 -0
package/dist/plugin-system/plugin-registry.d.ts +25 -0
package/dist/plugin-system/plugin-registry.js +167 -0
package/dist/plugin-system/plugin-types.d.ts +205 -0
package/dist/plugin-system/plugin-types.js +1 -0
package/dist/ports/index.d.ts +9 -0
package/dist/ports/index.js +1 -0
package/{src/report/sitegraph_template.ts → dist/report/crawl.html} +330 -81
package/dist/report/crawlExport.d.ts +3 -0
package/dist/report/{sitegraphExport.js → crawlExport.js} +3 -3
package/dist/report/crawl_template.d.ts +1 -0
package/dist/report/crawl_template.js +7 -0
package/dist/report/export.d.ts +3 -0
package/dist/report/export.js +81 -0
package/dist/report/html.js +15 -216
package/dist/report/insight.d.ts +27 -0
package/dist/report/insight.js +103 -0
package/dist/scoring/health.d.ts +56 -0
package/dist/scoring/health.js +213 -0
package/dist/utils/chalk.d.ts +6 -0
package/dist/utils/chalk.js +41 -0
package/dist/utils/secureConfig.d.ts +23 -0
package/dist/utils/secureConfig.js +128 -0
package/package.json +12 -6
package/CHANGELOG.md +0 -7
package/dist/db/schema.d.ts +0 -2
package/dist/graph/cluster.d.ts +0 -6
package/dist/graph/cluster.js +0 -173
package/dist/graph/duplicate.d.ts +0 -10
package/dist/graph/duplicate.js +0 -251
package/dist/report/sitegraphExport.d.ts +0 -3
package/dist/report/sitegraph_template.d.ts +0 -1
package/dist/report/sitegraph_template.js +0 -630
package/dist/scoring/hits.d.ts +0 -9
package/dist/scoring/hits.js +0 -111
package/src/analysis/analyze.ts +0 -548
package/src/analysis/content.ts +0 -62
package/src/analysis/images.ts +0 -28
package/src/analysis/links.ts +0 -41
package/src/analysis/scoring.ts +0 -59
package/src/analysis/seo.ts +0 -82
package/src/analysis/structuredData.ts +0 -62
package/src/audit/dns.ts +0 -49
package/src/audit/headers.ts +0 -98
package/src/audit/index.ts +0 -66
package/src/audit/scoring.ts +0 -232
package/src/audit/transport.ts +0 -258
package/src/audit/types.ts +0 -102
package/src/core/network/proxyAdapter.ts +0 -21
package/src/core/network/rateLimiter.ts +0 -39
package/src/core/network/redirectController.ts +0 -47
package/src/core/network/responseLimiter.ts +0 -34
package/src/core/network/retryPolicy.ts +0 -57
package/src/core/scope/domainFilter.ts +0 -45
package/src/core/scope/scopeManager.ts +0 -52
package/src/core/scope/subdomainPolicy.ts +0 -39
package/src/core/security/ipGuard.ts +0 -92
package/src/crawler/crawl.ts +0 -382
package/src/crawler/extract.ts +0 -34
package/src/crawler/fetcher.ts +0 -233
package/src/crawler/metricsRunner.ts +0 -124
package/src/crawler/normalize.ts +0 -108
package/src/crawler/parser.ts +0 -190
package/src/crawler/sitemap.ts +0 -73
package/src/crawler/trap.ts +0 -96
package/src/db/graphLoader.ts +0 -105
package/src/db/index.ts +0 -70
package/src/db/repositories/EdgeRepository.ts +0 -29
package/src/db/repositories/MetricsRepository.ts +0 -49
package/src/db/repositories/PageRepository.ts +0 -128
package/src/db/repositories/SiteRepository.ts +0 -32
package/src/db/repositories/SnapshotRepository.ts +0 -74
package/src/db/schema.ts +0 -177
package/src/diff/compare.ts +0 -84
package/src/graph/cluster.ts +0 -192
package/src/graph/duplicate.ts +0 -286
package/src/graph/graph.ts +0 -172
package/src/graph/metrics.ts +0 -110
package/src/graph/pagerank.ts +0 -125
package/src/graph/simhash.ts +0 -61
package/src/index.ts +0 -30
package/src/lock/hashKey.ts +0 -51
package/src/lock/lockManager.ts +0 -124
package/src/lock/pidCheck.ts +0 -13
package/src/report/html.ts +0 -227
package/src/report/sitegraphExport.ts +0 -58
package/src/scoring/hits.ts +0 -131
package/src/scoring/orphanSeverity.ts +0 -176
package/src/utils/version.ts +0 -18
package/tests/__snapshots__/orphanSeverity.test.ts.snap +0 -49
package/tests/analysis.unit.test.ts +0 -98
package/tests/analyze.integration.test.ts +0 -98
package/tests/audit/dns.test.ts +0 -31
package/tests/audit/headers.test.ts +0 -45
package/tests/audit/scoring.test.ts +0 -133
package/tests/audit/security.test.ts +0 -12
package/tests/audit/transport.test.ts +0 -112
package/tests/clustering.test.ts +0 -118
package/tests/crawler.test.ts +0 -358
package/tests/db.test.ts +0 -159
package/tests/diff.test.ts +0 -67
package/tests/duplicate.test.ts +0 -110
package/tests/fetcher.test.ts +0 -106
package/tests/fetcher_safety.test.ts +0 -85
package/tests/fixtures/analyze-crawl.json +0 -26
package/tests/hits.test.ts +0 -134
package/tests/html_report.test.ts +0 -58
package/tests/lock/lockManager.test.ts +0 -138
package/tests/metrics.test.ts +0 -196
package/tests/normalize.test.ts +0 -101
package/tests/orphanSeverity.test.ts +0 -160
package/tests/pagerank.test.ts +0 -98
package/tests/parser.test.ts +0 -117
package/tests/proxy_safety.test.ts +0 -57
package/tests/redirect_safety.test.ts +0 -73
package/tests/safety.test.ts +0 -114
package/tests/scope.test.ts +0 -66
package/tests/scoring.test.ts +0 -59
package/tests/sitemap.test.ts +0 -88
package/tests/soft404.test.ts +0 -41
package/tests/trap.test.ts +0 -39
package/tests/visualization_data.test.ts +0 -46
package/tsconfig.json +0 -11

package/tests/analysis.unit.test.ts DELETED Viewed

@@ -1,98 +0,0 @@
-import { describe, expect, test } from 'vitest';
-import { analyzeTitle, analyzeMetaDescription, applyDuplicateStatuses, analyzeH1 } from '../src/analysis/seo.js';
-import { analyzeContent, calculateThinContentScore } from '../src/analysis/content.js';
-import { analyzeStructuredData } from '../src/analysis/structuredData.js';
-import { analyzeLinks } from '../src/analysis/links.js';
-import { analyzeImageAlts } from '../src/analysis/images.js';
-describe('SEO module', () => {
-  test('analyze title edge cases', () => {
-    expect(analyzeTitle('<html></html>').status).toBe('missing');
-    expect(analyzeTitle('<title>short</title>').status).toBe('too_short');
-    expect(analyzeTitle(`<title>${'a'.repeat(61)}</title>`).status).toBe('too_long');
-    expect(analyzeTitle(`<title>${'a'.repeat(55)}</title>`).status).toBe('ok');
-  });
-  test('duplicate detection', () => {
-    const values = applyDuplicateStatuses([
-      { value: 'Same', length: 4, status: 'ok' as const },
-      { value: 'same', length: 4, status: 'ok' as const },
-      { value: null, length: 0, status: 'missing' as const }
-    ]);
-    expect(values[0].status).toBe('duplicate');
-    expect(values[1].status).toBe('duplicate');
-    expect(values[2].status).toBe('missing');
-  });
-  test('meta description boundaries', () => {
-    expect(analyzeMetaDescription('<meta name="description" content="">').status).toBe('missing');
-    expect(analyzeMetaDescription('<html></html>').status).toBe('missing');
-    expect(analyzeMetaDescription('<meta name="description" content="short">').status).toBe('too_short');
-    expect(analyzeMetaDescription(`<meta name="description" content="${'x'.repeat(150)}">`).status).toBe('ok');
-    expect(analyzeMetaDescription(`<meta name="description" content="${'x'.repeat(170)}">`).status).toBe('too_long');
-  });
-  test('h1 variations', () => {
-    expect(analyzeH1('<h1>One</h1>', 'Title').status).toBe('ok');
-    expect(analyzeH1('<h1>One</h1><h1>Two</h1>', 'Title').status).toBe('warning');
-    const noH1 = analyzeH1('<p>none</p>', 'Title');
-    expect(noH1.status).toBe('critical');
-    const same = analyzeH1('<h1>same</h1>', 'Same');
-    expect(same.matchesTitle).toBe(true);
-  });
-});
-describe('content module', () => {
-  test('word count strips nav/footer/script/style', () => {
-    const html = '<body><nav>skip me</nav><p>keep words here</p><footer>skip</footer><script>var x</script><style>.x{}</style></body>';
-    const result = analyzeContent(html);
-    expect(result.wordCount).toBe(3);
-    expect(result.uniqueSentenceCount).toBe(1);
-    expect(result.textHtmlRatio).toBeGreaterThan(0);
-  });
-  test('thin score boundaries', () => {
-    expect(calculateThinContentScore({ wordCount: 600, textHtmlRatio: 0.5, uniqueSentenceCount: 4 }, 0)).toBe(0);
-    expect(calculateThinContentScore({ wordCount: 0, textHtmlRatio: 0, uniqueSentenceCount: 1 }, 100)).toBe(100);
-  });
-  test('content handles malformed/empty html', () => {
-    expect(analyzeContent('').wordCount).toBe(0);
-    expect(analyzeContent('<div><span>broken').wordCount).toBeGreaterThanOrEqual(1);
-  });
-});
-describe('structured data', () => {
-  test('valid and invalid JSON-LD parsing', () => {
-    const valid = analyzeStructuredData('<script type="application/ld+json">{"@type":"Article"}</script>');
-    expect(valid.present).toBe(true);
-    expect(valid.valid).toBe(true);
-    expect(valid.types).toContain('Article');
-    const invalid = analyzeStructuredData('<script type="application/ld+json">{invalid}</script>');
-    expect(invalid.present).toBe(true);
-    expect(invalid.valid).toBe(false);
-    const missing = analyzeStructuredData('<p>none</p>');
-    expect(missing.present).toBe(false);
-  });
-});
-describe('links and images', () => {
-  test('link ratio calculation', () => {
-    const html = '<a href="/a">A</a><a href="https://other.com">B</a><a href="https://other.com" rel="nofollow">C</a>';
-    const links = analyzeLinks(html, 'https://example.com/page', 'https://example.com');
-    expect(links.internalLinks).toBe(1);
-    expect(links.externalLinks).toBe(2);
-    expect(links.nofollowCount).toBe(1);
-    expect(links.externalRatio).toBeCloseTo(2 / 3);
-  });
-  test('image alt detection', () => {
-    const html = '<img src="a"><img src="b" alt=""><img src="c" alt="ok">';
-    const imgs = analyzeImageAlts(html);
-    expect(imgs.totalImages).toBe(3);
-    expect(imgs.missingAlt).toBe(1);
-    expect(imgs.emptyAlt).toBe(1);
-  });
-});

package/tests/analyze.integration.test.ts DELETED Viewed

@@ -1,98 +0,0 @@
-import { describe, expect, test } from 'vitest';
-import path from 'node:path';
-import fs from 'node:fs/promises';
-import { analyzeSite, renderAnalysisHtml } from '../src/analysis/analyze.js';
-describe('analyze integration', () => {
-  const fixturePath = path.resolve(import.meta.dirname, 'fixtures/analyze-crawl.json');
-  test('analyzes full crawl fixture and schema', async () => {
-    const result = await analyzeSite('https://example.com', { fromCrawl: fixturePath });
-    expect(result.site_summary.pages_analyzed).toBe(3);
-    expect(result.site_summary.duplicate_titles).toBe(2);
-    expect(result.site_summary.avg_seo_score).toBeGreaterThanOrEqual(0);
-    expect(result.pages[0]).toHaveProperty('title');
-    expect(result.pages[0]).toHaveProperty('content');
-    expect(result.pages[0]).toHaveProperty('links');
-    expect(result.site_scores.overallScore).toBeGreaterThanOrEqual(0);
-    expect(result.site_scores.overallScore).toBeLessThanOrEqual(100);
-  });
-  test('module filter flags behavior', async () => {
-    const seoOnly = await analyzeSite('https://example.com', { fromCrawl: fixturePath, seo: true });
-    expect(seoOnly.pages[0].content.wordCount).toBe(0);
-    expect(seoOnly.pages[0].images.totalImages).toBe(0);
-    const contentOnly = await analyzeSite('https://example.com', { fromCrawl: fixturePath, content: true });
-    expect(contentOnly.pages[0].title.status).toBe('missing');
-    expect(contentOnly.pages[0].thinScore).toBeGreaterThanOrEqual(0);
-    const accessibilityOnly = await analyzeSite('https://example.com', { fromCrawl: fixturePath, accessibility: true });
-    expect(accessibilityOnly.pages[0].images.totalImages).toBeGreaterThan(0);
-    expect(accessibilityOnly.pages[0].title.status).toBe('missing');
-  });
-  test('html report generation', async () => {
-    const result = await analyzeSite('https://example.com', { fromCrawl: fixturePath });
-    const html = renderAnalysisHtml(result);
-    expect(html).toContain('<table');
-    expect(html).toContain('Analysis');
-  });
-  test('default database loading', async () => {
-    // Force in-memory DB for this test
-    process.env.CRAWLITH_DB_PATH = ':memory:';
-    // Close existing DB connection if any to ensure fresh start
-    const { getDb, closeDb } = await import('../src/db/index.js');
-    closeDb();
-    // Setup repositories
-    const { SiteRepository } = await import('../src/db/repositories/SiteRepository.js');
-    const { SnapshotRepository } = await import('../src/db/repositories/SnapshotRepository.js');
-    const { PageRepository } = await import('../src/db/repositories/PageRepository.js');
-    const db = getDb();
-    const siteRepo = new SiteRepository(db);
-    const snapshotRepo = new SnapshotRepository(db);
-    const pageRepo = new PageRepository(db);
-    // Create site and snapshot
-    const siteId = siteRepo.createSite('example.com');
-    const snapshotId = snapshotRepo.createSnapshot(siteId, 'full', 'running');
-    // Parse fixture and load pages into db
-    const rawYaml = await fs.readFile(fixturePath, 'utf-8');
-    const rawData = JSON.parse(rawYaml);
-    (rawData.pages || rawData.nodes).forEach((p: any) => {
-      pageRepo.upsertPage({
-        site_id: siteId,
-        normalized_url: p.url,
-        last_seen_snapshot_id: snapshotId,
-        http_status: p.status || 200,
-        html: p.html || '',
-        depth: p.depth || 0,
-      });
-    });
-    snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', { node_count: 3, edge_count: 0 });
-    try {
-      const result = await analyzeSite('https://example.com', {});
-      expect(result.site_summary.pages_analyzed).toBe(3);
-    } finally {
-      closeDb();
-      delete process.env.CRAWLITH_DB_PATH;
-    }
-  });
-  test('handles large html and js-only content', async () => {
-    const hugeText = '<html><body><script>document.write("x")</script>' + '<p>word </p>'.repeat(1000) + '</body></html>';
-    const tmpFile = path.resolve(import.meta.dirname, 'fixtures/large-analyze.json');
-    await fs.writeFile(tmpFile, JSON.stringify({ pages: [{ url: 'https://example.com/', status: 200, depth: 0, html: hugeText }] }));
-    const result = await analyzeSite('https://example.com', { fromCrawl: tmpFile });
-    expect(result.pages[0].content.wordCount).toBe(1000);
-    await fs.unlink(tmpFile);
-  });
-});

package/tests/audit/dns.test.ts DELETED Viewed

@@ -1,31 +0,0 @@
-import { describe, it, expect, vi } from 'vitest';
-import { resolveDns } from '../../src/audit/dns.js';
-import dns from 'node:dns/promises';
-vi.mock('node:dns/promises');
-describe('DNS Diagnostics', () => {
-  it('should resolve all records', async () => {
-    vi.spyOn(dns, 'resolve4').mockResolvedValue(['1.1.1.1']);
-    vi.spyOn(dns, 'resolve6').mockResolvedValue(['2606::1']);
-    vi.spyOn(dns, 'resolveCname').mockRejectedValue(new Error('ENODATA'));
-    vi.spyOn(dns, 'reverse').mockResolvedValue(['one.one.one.one']);
-    const result = await resolveDns('example.com');
-    expect(result.a).toEqual(['1.1.1.1']);
-    expect(result.aaaa).toEqual(['2606::1']);
-    expect(result.ipv6Support).toBe(true);
-    expect(result.reverse).toEqual(['one.one.one.one']);
-    expect(result.resolutionTime).toBeGreaterThanOrEqual(0);
-  });
-  it('should handle failures gracefully', async () => {
-    vi.spyOn(dns, 'resolve4').mockRejectedValue(new Error('ENOTFOUND'));
-    vi.spyOn(dns, 'resolve6').mockRejectedValue(new Error('ENOTFOUND'));
-    vi.spyOn(dns, 'resolveCname').mockRejectedValue(new Error('ENOTFOUND'));
-    const result = await resolveDns('invalid.com');
-    expect(result.a).toEqual([]);
-    expect(result.ipCount).toBe(0);
-  });
-});

package/tests/audit/headers.test.ts DELETED Viewed

@@ -1,45 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { analyzeHeaders } from '../../src/audit/headers.js';
-describe('Headers Analysis', () => {
-  it('should detect all secure headers', () => {
-    const headers = {
-      'strict-transport-security': 'max-age=31536000; includeSubDomains',
-      'content-security-policy': "default-src 'self'",
-      'x-frame-options': 'DENY',
-      'x-content-type-options': 'nosniff',
-      'referrer-policy': 'strict-origin-when-cross-origin',
-      'permissions-policy': 'geolocation=()'
-    };
-    const result = analyzeHeaders(headers);
-    expect(result.score).toBe(100);
-    expect(result.strictTransportSecurity.valid).toBe(true);
-  });
-  it('should handle missing headers', () => {
-    const headers = {};
-    const result = analyzeHeaders(headers);
-    expect(result.score).toBe(0);
-    expect(result.strictTransportSecurity.present).toBe(false);
-  });
-  it('should validate HSTS properly', () => {
-    const headers = {
-      'strict-transport-security': 'max-age=0'
-    };
-    // valid requires max-age
-    const result = analyzeHeaders(headers);
-    expect(result.strictTransportSecurity.valid).toBe(true);
-    // Wait, checkHSTS: includes('max-age=') is true. includes('includeSubDomains') is false.
-    // Issues will contain 'Missing includeSubDomains'.
-    expect(result.strictTransportSecurity.issues).toContain('Missing includeSubDomains');
-  });
-  it('should validate invalid HSTS', () => {
-    const headers = {
-      'strict-transport-security': 'invalid'
-    };
-    const result = analyzeHeaders(headers);
-    expect(result.strictTransportSecurity.valid).toBe(false);
-  });
-});

package/tests/audit/scoring.test.ts DELETED Viewed

@@ -1,133 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { calculateScore } from '../../src/audit/scoring.js';
-import { TransportDiagnostics, DnsDiagnostics, SecurityHeadersResult, PerformanceMetrics, AuditIssue } from '../../src/audit/types.js';
-describe('Scoring Engine', () => {
-  const mockTransport: TransportDiagnostics = {
-    tlsVersion: 'TLSv1.3',
-    cipherSuite: 'TLS_AES_256_GCM_SHA384',
-    alpnProtocol: 'h2',
-    certificate: {
-      issuer: 'Let\'s Encrypt',
-      subject: 'example.com',
-      validFrom: '2023-01-01',
-      validTo: '2024-01-01',
-      daysUntilExpiry: 60,
-      isSelfSigned: false,
-      isValidChain: true,
-      fingerprint: 'SHA256:...'
-    } as any,
-    httpVersion: '2.0',
-    compression: ['gzip'],
-    keepAlive: true,
-    transferEncoding: null,
-    redirectCount: 0,
-    redirects: [],
-    serverHeader: 'nginx',
-    headers: {}
-  };
-  const mockDns: DnsDiagnostics = {
-    a: ['1.1.1.1', '1.0.0.1'],
-    aaaa: ['2606:4700:4700::1111'],
-    cname: [],
-    reverse: [],
-    ipCount: 3,
-    ipv6Support: true,
-    resolutionTime: 10
-  };
-  const mockHeaders: SecurityHeadersResult = {
-    strictTransportSecurity: { present: true, valid: true, value: 'max-age=31536000' },
-    contentSecurityPolicy: { present: true, valid: true, value: "default-src 'self'" },
-    xFrameOptions: { present: true, valid: true, value: 'DENY' },
-    xContentTypeOptions: { present: true, valid: true, value: 'nosniff' },
-    referrerPolicy: { present: true, valid: true, value: 'strict-origin' },
-    permissionsPolicy: { present: true, valid: true, value: 'geolocation=()' },
-    details: {},
-    score: 100
-  };
-  const mockPerformance: PerformanceMetrics = {
-    dnsLookupTime: 10,
-    tcpConnectTime: 20,
-    tlsHandshakeTime: 30,
-    ttfb: 100,
-    totalTime: 200,
-    htmlSize: 50000,
-    headerSize: 500,
-    redirectTime: 0
-  };
-  it('should give perfect score for perfect inputs', () => {
-    const result = calculateScore(mockTransport, mockDns, mockHeaders, mockPerformance, []);
-    expect(result.score).toBe(100);
-    expect(result.grade).toBe('A');
-    expect(result.issues).toHaveLength(0);
-  });
-  it('should penalize TLS < 1.2', () => {
-    const badTransport = { ...mockTransport, tlsVersion: 'TLSv1.1' };
-    const result = calculateScore(badTransport, mockDns, mockHeaders, mockPerformance, []);
-    expect(result.score).toBeLessThan(100);
-    expect(result.categoryScores.transport).toBeLessThan(30);
-    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'tls-old' })]));
-  });
-  it('should penalize missing HTTPS', () => {
-    const badTransport = { ...mockTransport, tlsVersion: null, certificate: null };
-    const result = calculateScore(badTransport, mockDns, mockHeaders, mockPerformance, []);
-    expect(result.score).toBeLessThan(50); // Critical
-    expect(result.grade).toBe('F');
-    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'no-https' })]));
-  });
-  it('should fail on expired cert', () => {
-    const expiredTransport = {
-        ...mockTransport,
-        certificate: { ...mockTransport.certificate!, daysUntilExpiry: -5, validTo: '2023-01-01' }
-    };
-    const result = calculateScore(expiredTransport, mockDns, mockHeaders, mockPerformance, []);
-    expect(result.grade).toBe('F');
-    expect(result.score).toBeLessThanOrEqual(40);
-    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'cert-expired' })]));
-  });
-  it('should penalize missing security headers', () => {
-    // If score is 50, it means we lost 50 points in headers category (internal score)
-    // headers category is 20 points total. So we lose 10 points.
-    const badHeaders = { ...mockHeaders, score: 50, strictTransportSecurity: { present: false, valid: false, value: null } };
-    const result = calculateScore(mockTransport, mockDns, badHeaders, mockPerformance, []);
-    expect(result.categoryScores.security).toBe(10);
-    expect(result.score).toBe(90); // 100 - 10
-    expect(result.issues).toEqual(expect.arrayContaining([expect.objectContaining({ id: 'hsts-missing' })]));
-  });
-  it('should penalize poor performance', () => {
-      const badPerf = { ...mockPerformance, ttfb: 1000, htmlSize: 2000000 };
-      const result = calculateScore(mockTransport, mockDns, mockHeaders, badPerf, []);
-      // TTFB > 800: Lose 10 pts
-      // HTML > 1MB: Lose 5 pts
-      // Total perf score (30) -> 15.
-      expect(result.categoryScores.performance).toBe(15);
-      expect(result.score).toBe(85);
-      expect(result.issues).toEqual(expect.arrayContaining([
-          expect.objectContaining({ id: 'slow-ttfb' }),
-          expect.objectContaining({ id: 'large-html' })
-      ]));
-  });
-  it('should penalize infrastructure issues', () => {
-      const badDns = { ...mockDns, ipv6Support: false, ipCount: 1 };
-      const result = calculateScore(mockTransport, badDns, mockHeaders, mockPerformance, []);
-      // No IPv6: Lose 10 pts
-      // Single IP: Lose 10 pts
-      // Infra score (20) -> 0.
-      expect(result.categoryScores.infrastructure).toBe(0);
-      expect(result.score).toBe(80);
-      expect(result.issues).toEqual(expect.arrayContaining([
-          expect.objectContaining({ id: 'no-ipv6' }),
-          expect.objectContaining({ id: 'single-ip' })
-      ]));
-  });
-});

package/tests/audit/security.test.ts DELETED Viewed

@@ -1,12 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { auditUrl } from '../../src/audit/index.js';
-describe('Audit Security', () => {
-    it('should block audits of internal IP addresses', async () => {
-        await expect(auditUrl('http://127.0.0.1')).rejects.toThrow('Access to internal or private infrastructure is prohibited');
-    });
-    it('should block audits of link-local addresses', async () => {
-        await expect(auditUrl('http://169.254.169.254')).rejects.toThrow('Access to internal or private infrastructure is prohibited');
-    });
-});

package/tests/audit/transport.test.ts DELETED Viewed

@@ -1,112 +0,0 @@
-import { describe, it, expect, vi, afterEach } from 'vitest';
-import { analyzeTransport } from '../../src/audit/transport.js';
-import https from 'node:https';
-import http from 'node:http';
-import tls from 'node:tls';
-import { EventEmitter } from 'events';
-vi.mock('node:https');
-vi.mock('node:http');
-describe('Transport Diagnostics', () => {
-  afterEach(() => {
-    vi.clearAllMocks();
-  });
-  it('should analyze HTTPS transport', async () => {
-    // Mock Response
-    const mockRes = new EventEmitter() as any;
-    mockRes.statusCode = 200;
-    mockRes.statusMessage = 'OK';
-    mockRes.headers = {
-      'content-encoding': 'gzip',
-      'server': 'nginx',
-      'connection': 'keep-alive'
-    };
-    mockRes.httpVersion = '1.1';
-    const mockSocket = new EventEmitter();
-    Object.setPrototypeOf(mockSocket, tls.TLSSocket.prototype);
-    (mockSocket as any).getPeerCertificate = () => ({
-      subject: { CN: 'example.com' },
-      issuer: { CN: 'Let\'s Encrypt' },
-      valid_from: 'Jan 1 2023',
-      valid_to: 'Jan 1 2024',
-      fingerprint: 'SHA256:...'
-    });
-    (mockSocket as any).getProtocol = () => 'TLSv1.3';
-    (mockSocket as any).getCipher = () => ({ name: 'TLS_AES_...' });
-    (mockSocket as any).alpnProtocol = 'h2';
-    (mockSocket as any).authorized = true;
-    mockRes.socket = mockSocket;
-    // Mock Request
-    const mockReq = new EventEmitter() as any;
-    mockReq.end = vi.fn();
-    mockReq.destroy = vi.fn();
-    // Mock https.request
-    vi.spyOn(https, 'request').mockImplementation((url, options, cb) => {
-      if (cb) cb(mockRes);
-      // Simulate socket events
-      setTimeout(() => {
-        mockReq.emit('socket', mockRes.socket);
-        mockRes.socket.emit('lookup');
-        mockRes.socket.emit('connect');
-        mockRes.socket.emit('secureConnect');
-        mockReq.emit('finish');
-        // Response data
-        mockRes.emit('data', Buffer.from('<html></html>'));
-        mockRes.emit('end');
-      }, 10);
-      return mockReq;
-    });
-    const result = await analyzeTransport('https://example.com', 1000);
-    expect(result.transport.tlsVersion).toBe('TLSv1.3');
-    expect(result.transport.httpVersion).toBe('1.1');
-    expect(result.performance.htmlSize).toBeGreaterThan(0);
-    expect(result.transport.headers['server']).toBe('nginx');
-  });
-  it('should handle redirects', async () => {
-    const req1 = new EventEmitter() as any; req1.end = vi.fn(); req1.destroy = vi.fn();
-    const res1 = new EventEmitter() as any; res1.statusCode = 301; res1.headers = { location: 'https://example.com/' };
-    res1.socket = new EventEmitter(); Object.setPrototypeOf(res1.socket, tls.TLSSocket.prototype);
-    const req2 = new EventEmitter() as any; req2.end = vi.fn(); req2.destroy = vi.fn();
-    const res2 = new EventEmitter() as any; res2.statusCode = 200; res2.headers = {};
-    res2.socket = new EventEmitter(); Object.setPrototypeOf(res2.socket, tls.TLSSocket.prototype);
-    // Setup res2 socket for TLS checks
-    res2.socket.getPeerCertificate = () => ({});
-    res2.socket.getProtocol = () => 'TLSv1.2';
-    res2.socket.getCipher = () => ({ name: 'AES' });
-    const requestSpy = vi.spyOn(https, 'request');
-    requestSpy
-      .mockImplementationOnce((url, options, cb) => {
-        if (cb) cb(res1);
-        setTimeout(() => {
-           req1.emit('socket', res1.socket);
-           res1.emit('data', Buffer.from('redirecting'));
-           res1.emit('end');
-        }, 10);
-        return req1;
-      })
-      .mockImplementationOnce((url, options, cb) => {
-        if (cb) cb(res2);
-        setTimeout(() => {
-           req2.emit('socket', res2.socket);
-           res2.emit('data', Buffer.from('ok'));
-           res2.emit('end');
-        }, 10);
-        return req2;
-      });
-    const result = await analyzeTransport('https://redirect.com', 1000);
-    expect(result.transport.redirectCount).toBe(1);
-    expect(result.transport.redirects[0].location).toBe('https://example.com/');
-  });
-});

package/tests/clustering.test.ts DELETED Viewed

@@ -1,118 +0,0 @@
-import { describe, it, expect, beforeEach } from 'vitest';
-import { Graph } from '../src/graph/graph.js';
-import { detectContentClusters } from '../src/graph/cluster.js';
-describe('Content Clustering', () => {
-    let graph: Graph;
-    beforeEach(() => {
-        graph = new Graph();
-    });
-    it('should group similar pages into a cluster', () => {
-        // Mock simhashes for similar pages (Hamming distance 1)
-        const h1 = 0b101010n;
-        const h2 = 0b101011n;
-        const h3 = 0b101001n;
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-        graph.addNode('https://example.com/p3', 0, 200);
-        graph.updateNodeData('https://example.com/p1', { simhash: h1.toString() });
-        graph.updateNodeData('https://example.com/p2', { simhash: h2.toString() });
-        graph.updateNodeData('https://example.com/p3', { simhash: h3.toString() });
-        const clusters = detectContentClusters(graph, 2, 2);
-        expect(clusters.length).toBe(1);
-        expect(clusters[0].count).toBe(3);
-        expect(graph.nodes.get('https://example.com/p1')?.clusterId).toBe(1);
-    });
-    it('should separate dissimilar pages', () => {
-        // Mock simhashes for very different pages
-        const h1 = 0b1111111111n;
-        const h2 = 0b0000000000n;
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-        graph.updateNodeData('https://example.com/p1', { simhash: h1.toString() });
-        graph.updateNodeData('https://example.com/p2', { simhash: h2.toString() });
-        const clusters = detectContentClusters(graph, 2, 2);
-        expect(clusters.length).toBe(0); // None meet minSize 2
-    });
-    it('should respect minClusterSize', () => {
-        const h1 = 0b1n;
-        const h2 = 0b0n;
-        graph.addNode('https://example.com/p1', 0, 200);
-        graph.addNode('https://example.com/p2', 0, 200);
-        graph.updateNodeData('https://example.com/p1', { simhash: h1.toString() });
-        graph.updateNodeData('https://example.com/p2', { simhash: h2.toString() });
-        const clusters = detectContentClusters(graph, 1, 3);
-        expect(clusters.length).toBe(0);
-    });
-    it('should identify shared path prefixes (silos)', () => {
-        graph.addNode('https://example.com/blog/seo-tips', 0, 200);
-        graph.addNode('https://example.com/blog/link-building', 0, 200);
-        graph.addNode('https://example.com/blog/technical-seo', 0, 200);
-        const h = 0b111n;
-        graph.updateNodeData('https://example.com/blog/seo-tips', { simhash: h.toString() });
-        graph.updateNodeData('https://example.com/blog/link-building', { simhash: h.toString() });
-        graph.updateNodeData('https://example.com/blog/technical-seo', { simhash: h.toString() });
-        const clusters = detectContentClusters(graph, 0, 3);
-        expect(clusters[0].sharedPathPrefix).toBe('/blog');
-    });
-    it('should be deterministic with unstable input order', () => {
-        // We'll add nodes in different orders and check if cluster primary is same
-        const h = 0b111n;
-        graph.addNode('https://example.com/z', 0, 200);
-        graph.addNode('https://example.com/a', 0, 200);
-        graph.addNode('https://example.com/m', 0, 200);
-        graph.updateNodeData('https://example.com/z', { simhash: h.toString(), pageRank: 10 });
-        graph.updateNodeData('https://example.com/a', { simhash: h.toString(), pageRank: 10 });
-        graph.updateNodeData('https://example.com/m', { simhash: h.toString(), pageRank: 10 });
-        const clusters = detectContentClusters(graph, 0, 3);
-        // a should be primary because it's shortest/lexicographic first since PageRanks are same
-        expect(clusters[0].primaryUrl).toBe('https://example.com/a');
-    });
-    it('should use band optimization correctly (heuristic nature)', () => {
-        // Create many nodes in 2 groups
-        // Group 1: Matches in band 0
-        // Group 2: Matches in band 1
-        for (let i = 0; i < 5; i++) {
-            const url = `https://example.com/g1/${i}`;
-            graph.addNode(url, 0, 200);
-            // Simhash that matches in first 16 bits (0xAAAA)
-            const hash = BigInt(0xAAAA) | (BigInt(i) << 16n);
-            graph.updateNodeData(url, { simhash: hash.toString() });
-        }
-        for (let i = 0; i < 5; i++) {
-            const url = `https://example.com/g2/${i}`;
-            graph.addNode(url, 0, 200);
-            // Simhash that matches in second 16 bits (0xBBBB << 16)
-            const hash = (BigInt(0xBBBB) << 16n) | BigInt(i);
-            graph.updateNodeData(url, { simhash: hash.toString() });
-        }
-        const clusters = detectContentClusters(graph, 5, 3);
-        expect(clusters.length).toBe(2);
-        expect(clusters[0].count).toBe(5);
-        expect(clusters[1].count).toBe(5);
-    });
-});