@crawlith/core 0.1.0 → 0.1.2

package/tests/metrics.test.ts

@@ -1,196 +0,0 @@
-import { Graph } from '../src/graph/graph.js';
-import { calculateMetrics } from '../src/graph/metrics.js';
-import { test, expect } from 'vitest';
-
-test('graph metrics basic', () => {
-  const g = new Graph();
-
-  // Structure:
-  // A -> B
-  // A -> C
-  // B -> C
-  // C -> A
-
-  g.addNode('A', 0, 200);
-  g.addNode('B', 1, 200);
-  g.addNode('C', 1, 200);
-
-  g.addEdge('A', 'B');
-  g.addEdge('A', 'C');
-  g.addEdge('B', 'C');
-  g.addEdge('C', 'A');
-
-  const metrics = calculateMetrics(g, 5);
-
-  expect(metrics.totalPages).toBe(3);
-  expect(metrics.totalEdges).toBe(4);
-
-  // Check degrees on nodes directly
-  const nodeA = g.nodes.get('A');
-  expect(nodeA?.inLinks).toBe(1);
-  expect(nodeA?.outLinks).toBe(2);
-
-  const nodeC = g.nodes.get('C');
-  expect(nodeC?.inLinks).toBe(2);
-  expect(nodeC?.outLinks).toBe(1);
-
-  expect(metrics.averageOutDegree).toBeCloseTo(4/3);
-
-  // Top authority should be C with 2 in-links, authority = 1
-  expect(metrics.topAuthorityPages[0].url).toBe('C');
-  expect(metrics.topAuthorityPages[0].authority).toBeCloseTo(1);
-
-  // Max depth found
-  expect(metrics.maxDepthFound).toBe(1);
-
-  // Orphan pages (none)
-  expect(metrics.orphanPages).toEqual([]);
-});
-
-test('orphan pages', () => {
-  const g = new Graph();
-  g.addNode('Root', 0, 200);
-  g.addNode('Orphan', 1, 200);
-  // Orphan is at depth 1 but no incoming edges recorded (maybe missed or filtered)
-
-  const metrics = calculateMetrics(g, 5);
-  expect(metrics.orphanPages).toContain('Orphan');
-  expect(metrics.orphanPages).not.toContain('Root');
-});
-test('metrics v2 calculations', () => {
-  const g = new Graph();
-
-  // Root (depth 0, in=0, out=2)
-  g.addNode('root', 0, 200);
-
-  // A (depth 1, in=1, out=1)
-  g.addNode('A', 1, 200);
-  g.addEdge('root', 'A');
-
-  // B (depth 1, in=1, out=0)
-  g.addNode('B', 1, 200);
-  g.addEdge('root', 'B');
-
-  // C (depth 2, in=1, out=0)
-  g.addNode('C', 2, 200);
-  g.addEdge('A', 'C');
-
-  // Orphan (depth 1, in=0) - e.g. added but no edge to it?
-  // If it's in graph with depth > 0 and inLinks=0, it's an orphan.
-  g.addNode('orphan', 1, 200);
-
-  // Near Orphan (depth 3, in=1)
-  g.addNode('D', 2, 200);
-  g.addNode('nearOrphan', 3, 200);
-  g.addEdge('C', 'D'); // C->D
-  g.addEdge('D', 'nearOrphan'); // D->nearOrphan
-
-  // Deep page (depth 4)
-  g.addNode('deep', 4, 200);
-  g.addEdge('nearOrphan', 'deep');
-
-  // Nodes: root(0), A(1), B(1), C(2), orphan(1), D(2), nearOrphan(3), deep(4)
-  // Total pages: 8
-
-  // Edges: root->A, root->B, A->C, C->D, D->nearOrphan, nearOrphan->deep
-  // Total edges: 6
-
-  // InLinks:
-  // root: 0
-  // A: 1
-  // B: 1
-  // C: 1
-  // orphan: 0
-  // D: 1
-  // nearOrphan: 1
-  // deep: 1
-
-  // Max InLinks = 1.
-  // Authority Score = log(1 + in) / log(1 + maxIn)
-  // If maxIn = 1, log(2).
-  // For A: log(2)/log(2) = 1.
-  // For root: log(1)/log(2) = 0.
-
-  // Let's make maxIn > 1 to test better.
-  g.addNode('popular', 1, 200);
-  g.addEdge('root', 'popular');
-  g.addEdge('A', 'popular');
-  // popular inLinks = 2. MaxIn = 2.
-  // Authority popular = log(3)/log(3) = 1.
-  // Authority A = log(2)/log(3) approx 0.63
-
-  const metrics = calculateMetrics(g, 10); // maxDepth arg (not used for calculation logic of deepPages which is hardcoded >=4 per prompt?)
-  // Prompt says "deepPages: depth >= 4".
-  // Existing calculateMetrics takes maxDepth arg.
-  // Existing: deepPages = nodes.filter(n => n.depth >= maxDepth)
-  // New requirement: deepPages: depth >= 4.
-  // I should probably ignore the argument or update the requirement interpretation.
-  // "deepPages: depth >= 4" implies fixed threshold.
-
-  // Orphan pages: inLinks === 0 && depth > 0
-  expect(metrics.orphanPages).toContain('orphan');
-  expect(metrics.orphanPages).not.toContain('root'); // depth 0
-
-  // Near orphans: inLinks === 1 && depth >= 3
-  expect(metrics.nearOrphans).toContain('nearOrphan'); // depth 3, in 1
-  expect(metrics.nearOrphans).toContain('deep'); // depth 4, in 1 (from nearOrphan)
-  expect(metrics.nearOrphans).not.toContain('D'); // depth 2
-
-  // Deep pages: depth >= 4
-  expect(metrics.deepPages).toContain('deep');
-  expect(metrics.deepPages).not.toContain('nearOrphan');
-
-  // Crawl Efficiency Score: 1 - (deepPagesCount / totalPages)
-  // Total: 9 nodes (root, A, B, C, orphan, D, nearOrphan, deep, popular)
-  // Deep: 1 (deep)
-  // Score: 1 - 1/9 = 8/9 = 0.888...
-  expect(metrics.crawlEfficiencyScore).toBeCloseTo(8/9);
-
-  // Average Depth: sum(depth) / totalPages
-  // Depths: 0, 1, 1, 2, 1, 2, 3, 4, 1
-  // Sum: 15
-  // Avg: 15/9 = 1.666...
-  expect(metrics.averageDepth).toBeCloseTo(15/9);
-
-  // Structural Entropy
-  // OutDegrees:
-  // root: 3 (A, B, popular)
-  // A: 2 (C, popular)
-  // B: 0
-  // C: 1 (D)
-  // orphan: 0
-  // D: 1 (nearOrphan)
-  // nearOrphan: 1 (deep)
-  // deep: 0
-  // popular: 0
-
-  // Distribution:
-  // 0: 4 nodes (B, orphan, deep, popular)
-  // 1: 3 nodes (C, D, nearOrphan)
-  // 2: 1 node (A)
-  // 3: 1 node (root)
-
-  // P(0) = 4/9
-  // P(1) = 3/9
-  // P(2) = 1/9
-  // P(3) = 1/9
-
-  // Entropy = - (4/9 log2(4/9) + 3/9 log2(3/9) + 1/9 log2(1/9) + 1/9 log2(1/9))
-  // = - (0.444 * -1.17 + 0.333 * -1.58 + 0.111 * -3.17 + 0.111 * -3.17)
-  // approx 1.75
-
-  // Let's compute exact expected value
-  const p0 = 4/9;
-  const p1 = 3/9;
-  const p2 = 1/9;
-  const p3 = 1/9;
-  const entropy = - (p0 * Math.log2(p0) + p1 * Math.log2(p1) + p2 * Math.log2(p2) + p3 * Math.log2(p3));
-
-  expect(metrics.structuralEntropy).toBeCloseTo(entropy);
-
-  // Limit Reached
-  expect(metrics.limitReached).toBe(false);
-  g.limitReached = true;
-  const metrics2 = calculateMetrics(g, 10);
-  expect(metrics2.limitReached).toBe(true);
-});

package/tests/normalize.test.ts

@@ -1,101 +0,0 @@
-import { normalizeUrl } from '../src/crawler/normalize.js';
-import { extractLinks } from '../src/crawler/extract.js';
-import { test, expect } from 'vitest';
-
-test('normalizeUrl', () => {
-  expect(normalizeUrl('https://Example.com/Foo/', '')).toBe('https://example.com/Foo');
-  expect(normalizeUrl('http://example.com:80/bar', '')).toBe('http://example.com/bar');
-  expect(normalizeUrl('https://example.com/baz#frag', '')).toBe('https://example.com/baz');
-  expect(normalizeUrl('https://example.com/qux?a=1', '', { stripQuery: true })).toBe('https://example.com/qux');
-  expect(normalizeUrl('https://example.com/qux?a=1', '', { stripQuery: false })).toBe('https://example.com/qux?a=1');
-  expect(normalizeUrl('https://example.com/', '')).toBe('https://example.com/');
-});
-
-test('extractLinks', () => {
-  const html = `
-    <html>
-      <body>
-        <a href="/foo">Foo</a>
-        <a href="bar">Bar</a>
-        <a href="https://other.com/baz">Baz</a>
-        <a href="#top">Top</a>
-      </body>
-    </html>
-  `;
-  const links = extractLinks(html, 'https://example.com/page/');
-  expect(links).toContain('https://example.com/foo');
-  expect(links).toContain('https://example.com/page/bar');
-  expect(links).toContain('https://other.com/baz');
-  expect(links).not.toContain('https://example.com/page/#top');
-  expect(links).toContain('https://example.com/page/'); // #top resolves to base url without fragment
-});
-test('normalizeUrl: absolute resolution', () => {
-  expect(normalizeUrl('/foo', 'https://example.com')).toBe('https://example.com/foo');
-  expect(normalizeUrl('bar', 'https://example.com/baz/')).toBe('https://example.com/baz/bar');
-  expect(normalizeUrl('//other.com/foo', 'https://example.com')).toBe('https://other.com/foo');
-});
-
-test('normalizeUrl: only http/https', () => {
-  expect(normalizeUrl('ftp://example.com/file', 'https://example.com')).toBeNull();
-  expect(normalizeUrl('mailto:user@example.com', 'https://example.com')).toBeNull();
-  expect(normalizeUrl('javascript:alert(1)', 'https://example.com')).toBeNull();
-});
-
-test('normalizeUrl: lowercase hostname', () => {
-  expect(normalizeUrl('https://EXAMPLE.com/foo', '')).toBe('https://example.com/foo');
-});
-
-test('normalizeUrl: remove default ports', () => {
-  expect(normalizeUrl('http://example.com:80/foo', '')).toBe('http://example.com/foo');
-  expect(normalizeUrl('https://example.com:443/foo', '')).toBe('https://example.com/foo');
-  expect(normalizeUrl('http://example.com:8080/foo', '')).toBe('http://example.com:8080/foo');
-});
-
-test('normalizeUrl: remove hash fragments', () => {
-  expect(normalizeUrl('https://example.com/foo#bar', '')).toBe('https://example.com/foo');
-});
-
-test('normalizeUrl: strip query', () => {
-  expect(normalizeUrl('https://example.com/foo?a=1&b=2', '', { stripQuery: true })).toBe('https://example.com/foo');
-});
-
-test('normalizeUrl: filter tracking params', () => {
-  const url = 'https://example.com/foo?utm_source=google&utm_medium=cpc&a=1&fbclid=123';
-  expect(normalizeUrl(url, '', { stripQuery: false })).toBe('https://example.com/foo?a=1');
-
-  const url2 = 'https://example.com/foo?gclid=abc&msclkid=def';
-  expect(normalizeUrl(url2, '', { stripQuery: false })).toBe('https://example.com/foo');
-});
-
-test('normalizeUrl: trailing slash', () => {
-  expect(normalizeUrl('https://example.com/foo/', '')).toBe('https://example.com/foo');
-  expect(normalizeUrl('https://example.com/', '')).toBe('https://example.com/');
-});
-
-test('normalizeUrl: collapse duplicate slashes', () => {
-  expect(normalizeUrl('https://example.com/foo//bar', '')).toBe('https://example.com/foo/bar');
-  expect(normalizeUrl('https://example.com//foo///bar', '')).toBe('https://example.com/foo/bar');
-});
-
-test('normalizeUrl: skip non-HTML assets', () => {
-  expect(normalizeUrl('https://example.com/file.pdf', '')).toBeNull();
-  expect(normalizeUrl('https://example.com/image.jpg', '')).toBeNull();
-  expect(normalizeUrl('https://example.com/image.png', '')).toBeNull();
-  expect(normalizeUrl('https://example.com/image.svg', '')).toBeNull();
-  expect(normalizeUrl('https://example.com/image.webp', '')).toBeNull();
-  expect(normalizeUrl('https://example.com/image.gif', '')).toBeNull();
-  expect(normalizeUrl('https://example.com/archive.zip', '')).toBeNull();
-  expect(normalizeUrl('https://example.com/data.xml', '')).toBeNull();
-  expect(normalizeUrl('https://example.com/data.json', '')).toBeNull();
-  expect(normalizeUrl('https://example.com/video.mp4', '')).toBeNull();
-
-  // HTML extensions should pass (or no extension)
-  expect(normalizeUrl('https://example.com/page.html', '')).toBe('https://example.com/page.html');
-  expect(normalizeUrl('https://example.com/page.htm', '')).toBe('https://example.com/page.htm');
-  expect(normalizeUrl('https://example.com/page', '')).toBe('https://example.com/page');
-});
-
-test('normalizeUrl: return format', () => {
-  const res = normalizeUrl('https://example.com/foo?a=1', '');
-  expect(res).toBe('https://example.com/foo?a=1');
-});

package/tests/orphanSeverity.test.ts

@@ -1,160 +0,0 @@
-import { describe, expect, test } from 'vitest';
-import { annotateOrphans, calculateOrphanSeverity, mapImpactLevel, type SitegraphNode, type SitegraphEdge } from '../src/scoring/orphanSeverity.js';
-
-function baseNode(url: string, overrides: Partial<SitegraphNode> = {}): SitegraphNode {
-  return {
-    url,
-    depth: 1,
-    inLinks: 0,
-    outLinks: 0,
-    status: 200,
-    ...overrides
-  };
-}
-
-describe('orphan detection and severity scoring', () => {
-  test('hard orphan detection and homepage exclusion', () => {
-    const nodes: SitegraphNode[] = [
-      baseNode('https://example.com/', { depth: 0, inLinks: 0 }),
-      baseNode('https://example.com/orphan', { inLinks: 0 })
-    ];
-    const edges: SitegraphEdge[] = [];
-
-    const result = annotateOrphans(nodes, edges, {
-      enabled: true,
-      severityEnabled: false,
-      includeSoftOrphans: false,
-      minInbound: 2,
-      rootUrl: 'https://example.com/'
-    });
-
-    expect(result[0]).toMatchObject({ orphan: false });
-    expect(result[1]).toMatchObject({ orphan: true, orphanType: 'hard' });
-  });
-
-  test('near orphan threshold override', () => {
-    const nodes = [baseNode('https://example.com/near', { inLinks: 2 })];
-    const edges: SitegraphEdge[] = [];
-
-    const resultDefault = annotateOrphans(nodes, edges, {
-      enabled: true,
-      severityEnabled: false,
-      includeSoftOrphans: false,
-      minInbound: 2
-    });
-    const resultStrict = annotateOrphans(nodes, edges, {
-      enabled: true,
-      severityEnabled: false,
-      includeSoftOrphans: false,
-      minInbound: 1
-    });
-
-    expect(resultDefault[0]).toMatchObject({ orphan: true, orphanType: 'near' });
-    expect(resultStrict[0]).toMatchObject({ orphan: false });
-  });
-
-  test('soft orphan detection only when enabled and inbound only from low-value sources', () => {
-    const nodes: SitegraphNode[] = [
-      baseNode('https://example.com/tag/seo', { pageType: 'tag', outLinks: 1 }),
-      baseNode('https://example.com/list?page=2', { pageType: 'pagination', outLinks: 1 }),
-      baseNode('https://example.com/target', { inLinks: 2 }),
-      baseNode('https://example.com/normal', { outLinks: 1 })
-    ];
-
-    const edges: SitegraphEdge[] = [
-      { source: 'https://example.com/tag/seo', target: 'https://example.com/target' },
-      { source: 'https://example.com/list?page=2', target: 'https://example.com/target' }
-    ];
-
-    const withSoft = annotateOrphans(nodes, edges, {
-      enabled: true,
-      severityEnabled: false,
-      includeSoftOrphans: true,
-      minInbound: 1
-    });
-
-    const withoutSoft = annotateOrphans(nodes, edges, {
-      enabled: true,
-      severityEnabled: false,
-      includeSoftOrphans: false,
-      minInbound: 1
-    });
-
-    expect(withSoft.find((n) => n.url.endsWith('/target'))).toMatchObject({ orphan: true, orphanType: 'soft' });
-    expect(withoutSoft.find((n) => n.url.endsWith('/target'))).toMatchObject({ orphan: false });
-  });
-
-  test('crawl-only orphan detection', () => {
-    const nodes = [baseNode('https://example.com/sitemap-only', { inLinks: 0, discoveredViaSitemap: true })];
-    const result = annotateOrphans(nodes, [], {
-      enabled: true,
-      severityEnabled: false,
-      includeSoftOrphans: false,
-      minInbound: 2
-    });
-
-    expect(result[0]).toMatchObject({ orphan: true, orphanType: 'crawl-only' });
-  });
-
-  test('severity calculation modifiers and score clamping', () => {
-    const high = calculateOrphanSeverity('hard', baseNode('https://example.com/high', {
-      inLinks: 0,
-      wordCount: 1500,
-      hasStructuredData: true,
-      depth: 1,
-      isProductOrCommercial: true
-    }));
-
-    const low = calculateOrphanSeverity('hard', baseNode('https://example.com/low', {
-      inLinks: 0,
-      wordCount: 120,
-      noindex: true,
-      duplicateContent: true,
-      pageType: 'archive'
-    }));
-
-    expect(high).toBe(100);
-    expect(low).toBe(80);
-  });
-
-  test('impact level mapping', () => {
-    expect(mapImpactLevel(0)).toBe('low');
-    expect(mapImpactLevel(39)).toBe('low');
-    expect(mapImpactLevel(40)).toBe('medium');
-    expect(mapImpactLevel(69)).toBe('medium');
-    expect(mapImpactLevel(70)).toBe('high');
-    expect(mapImpactLevel(89)).toBe('high');
-    expect(mapImpactLevel(90)).toBe('critical');
-    expect(mapImpactLevel(100)).toBe('critical');
-  });
-
-  test('canonical consolidation, robots exclusion, and deterministic JSON output snapshot', () => {
-    const nodes: SitegraphNode[] = [
-      baseNode('https://example.com/canonical', { inLinks: 0 }),
-      baseNode('https://example.com/variant?a=1', { canonicalUrl: 'https://example.com/canonical', inLinks: 1 }),
-      baseNode('https://example.com/blocked', { inLinks: 0, robotsExcluded: true }),
-      baseNode('https://example.com/redirect-target', { inLinks: 1 })
-    ];
-
-    const edges: SitegraphEdge[] = [
-      { source: 'https://example.com/redirect-source', target: 'https://example.com/redirect-target' }
-    ];
-
-    const options = {
-      enabled: true,
-      severityEnabled: true,
-      includeSoftOrphans: true,
-      minInbound: 2
-    };
-
-    const first = annotateOrphans(nodes, edges, options);
-    const second = annotateOrphans(nodes, edges, options);
-
-    expect(first).toEqual(second);
-    expect(first.find((n) => n.url.endsWith('/canonical'))).toMatchObject({ orphan: true, orphanType: 'near' });
-    expect(first.find((n) => n.url.endsWith('/blocked'))).toMatchObject({ orphan: false });
-
-    const normalized = JSON.stringify(first, null, 2).replace(/\r\n/g, '\n');
-    expect(normalized).toMatchSnapshot();
-  });
-});

package/tests/pagerank.test.ts

@@ -1,98 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { Graph } from '../src/graph/graph.js';
-import { computePageRank } from '../src/graph/pagerank.js';
-
-describe('PageRank Engine', () => {
-    it('should calculate identical PageRank for a simple loop', () => {
-        const graph = new Graph();
-        graph.addNode('https://a.com', 0, 200);
-        graph.addNode('https://b.com', 1, 200);
-        graph.addEdge('https://a.com', 'https://b.com');
-        graph.addEdge('https://b.com', 'https://a.com');
-
-        computePageRank(graph);
-        const nodes = graph.getNodes();
-
-        expect(nodes[0].pageRank).toBeCloseTo(0.5, 4);
-        expect(nodes[1].pageRank).toBeCloseTo(0.5, 4);
-        expect(nodes[0].pageRankScore).toBe(100);
-        expect(nodes[1].pageRankScore).toBe(100);
-    });
-
-    it('should identify the center of a star graph as most important', () => {
-        const graph = new Graph();
-        graph.addNode('https://center.com', 0, 200);
-        graph.addNode('https://p1.com', 1, 200);
-        graph.addNode('https://p2.com', 1, 200);
-        graph.addNode('https://p3.com', 1, 200);
-
-        // Star in: all link to center
-        graph.addEdge('https://p1.com', 'https://center.com');
-        graph.addEdge('https://p2.com', 'https://center.com');
-        graph.addEdge('https://p3.com', 'https://center.com');
-
-        computePageRank(graph);
-        const nodes = graph.getNodes();
-
-        const center = nodes.find(n => n.url.includes('center'))!;
-        const leaves = nodes.filter(n => !n.url.includes('center'));
-
-        expect(center.pageRankScore).toBe(100);
-        leaves.forEach(leaf => {
-            expect(leaf.pageRankScore).toBeLessThan(100);
-            expect(leaf.pageRank!).toBeLessThan(center.pageRank!);
-        });
-    });
-
-    it('should respect link weights (Body > Nav > Footer)', () => {
-        const graph = new Graph();
-        graph.addNode('https://source.com', 0, 200);
-        graph.addNode('https://body-target.com', 1, 200);
-        graph.addNode('https://footer-target.com', 1, 200);
-
-        // Body weight 1.0, Footer weight 0.4
-        graph.addEdge('https://source.com', 'https://body-target.com', 1.0);
-        graph.addEdge('https://source.com', 'https://footer-target.com', 0.4);
-
-        computePageRank(graph);
-
-        const bodyTarget = graph.nodes.get('https://body-target.com')!;
-        const footerTarget = graph.nodes.get('https://footer-target.com')!;
-
-        expect(bodyTarget.pageRank!).toBeGreaterThan(footerTarget.pageRank!);
-    });
-
-    it('should handle sink nodes by redistributing rank', () => {
-        const graph = new Graph();
-        graph.addNode('https://a.com', 0, 200);
-        graph.addNode('https://b.com', 1, 200); // b is a sink
-        graph.addEdge('https://a.com', 'https://b.com');
-
-        computePageRank(graph);
-
-        const nodeA = graph.nodes.get('https://a.com')!;
-        const nodeB = graph.nodes.get('https://b.com')!;
-
-        // Without redistribution, A would lose all rank.
-        // With redistribution, A should still have some rank.
-        expect(nodeA.pageRank).toBeGreaterThan(0);
-        expect(nodeB.pageRank).toBeGreaterThan(nodeA.pageRank!);
-    });
-
-    it('should exclude noindex pages from receiving or passing rank', () => {
-        const graph = new Graph();
-        graph.addNode('https://a.com', 0, 200);
-        graph.addNode('https://no-index.com', 1, 200);
-        graph.nodes.get('https://no-index.com')!.noindex = true;
-
-        graph.addEdge('https://a.com', 'https://no-index.com');
-
-        computePageRank(graph);
-
-        const nodeA = graph.nodes.get('https://a.com')!;
-        const nodeNoIndex = graph.nodes.get('https://no-index.com')!;
-
-        expect(nodeNoIndex.pageRank).toBeUndefined();
-        expect(nodeA.pageRank).toBe(1.0); // Only one eligible node
-    });
-});

package/tests/parser.test.ts

@@ -1,117 +0,0 @@
-import { test, expect } from 'vitest';
-import { Parser } from '../src/crawler/parser.js';
-
-const parser = new Parser();
-const baseUrl = 'https://example.com';
-
-test('extracts links correctly', () => {
-  const html = `
-    <html>
-      <body>
-        <a href="/page1">Page 1</a>
-        <a href="https://other.com">Other</a>
-        <a href="#hash">Hash</a>
-        <a href="javascript:void(0)">JS</a>
-      </body>
-    </html>
-  `;
-  const result = parser.parse(html, baseUrl, 200);
-  const urls = result.links.map(l => l.url);
-  expect(urls).toContain('https://example.com/page1');
-  expect(urls).toContain('https://other.com/');
-  expect(urls).not.toContain('https://example.com/#hash');
-  // It also extracts the base URL itself from href="#hash"
-  expect(urls).toContain('https://example.com/');
-  expect(result.links.length).toBe(3);
-});
-
-test('respects nofollow on links', () => {
-  const html = `
-    <html>
-      <body>
-        <a href="/page1" rel="nofollow">Page 1</a>
-        <a href="/page2">Page 2</a>
-      </body>
-    </html>
-  `;
-  const result = parser.parse(html, baseUrl, 200);
-  const urls = result.links.map(l => l.url);
-  expect(urls).not.toContain('https://example.com/page1');
-  expect(urls).toContain('https://example.com/page2');
-});
-
-test('respects meta robots nofollow', () => {
-  const html = `
-    <html>
-      <head>
-        <meta name="robots" content="nofollow">
-      </head>
-      <body>
-        <a href="/page1">Page 1</a>
-      </body>
-    </html>
-  `;
-  const result = parser.parse(html, baseUrl, 200);
-  expect(result.nofollow).toBe(true);
-  expect(result.links.length).toBe(0);
-});
-
-test('detects canonical', () => {
-  const html = `
-    <html>
-      <head>
-        <link rel="canonical" href="https://example.com/canon">
-      </head>
-    </html>
-  `;
-  const result = parser.parse(html, baseUrl, 200);
-  expect(result.canonical).toBe('https://example.com/canon');
-});
-
-test('detects relative canonical', () => {
-  const html = `
-    <html>
-      <head>
-        <link rel="canonical" href="/canon">
-      </head>
-    </html>
-  `;
-  const result = parser.parse(html, baseUrl, 200);
-  expect(result.canonical).toBe('https://example.com/canon');
-});
-
-test('detects soft 404', () => {
-  const html = `
-    <html>
-      <head><title>Page Not Found</title></head>
-      <body>Sorry, the page you are looking for does not exist.</body>
-    </html>
-  `;
-  const result = parser.parse(html, baseUrl, 200);
-  expect(result.soft404Score).toBeGreaterThanOrEqual(0.5);
-});
-
-test('content hash ignores scripts', () => {
-  const html1 = `
-    <html><body><script>var x=1;</script><p>Hello</p></body></html>
-  `;
-  const html2 = `
-    <html><body><script>var x=2;</script><p>Hello</p></body></html>
-  `;
-  const result1 = parser.parse(html1, baseUrl, 200);
-  const result2 = parser.parse(html2, baseUrl, 200);
-  expect(result1.contentHash).toBe(result2.contentHash);
-});
-
-test('detects meta robots noindex', () => {
-  const html = `
-    <html>
-      <head>
-        <meta name="robots" content="noindex, nofollow">
-      </head>
-    </html>
-  `;
-  const result = parser.parse(html, baseUrl, 200);
-  expect(result.noindex).toBe(true);
-  expect(result.nofollow).toBe(true);
-});