@crawlith/core 0.1.0

package/tests/metrics.test.ts

@@ -0,0 +1,196 @@
+import { Graph } from '../src/graph/graph.js';
+import { calculateMetrics } from '../src/graph/metrics.js';
+import { test, expect } from 'vitest';
+
+test('graph metrics basic', () => {
+  const g = new Graph();
+
+  // Structure:
+  // A -> B
+  // A -> C
+  // B -> C
+  // C -> A
+
+  g.addNode('A', 0, 200);
+  g.addNode('B', 1, 200);
+  g.addNode('C', 1, 200);
+
+  g.addEdge('A', 'B');
+  g.addEdge('A', 'C');
+  g.addEdge('B', 'C');
+  g.addEdge('C', 'A');
+
+  const metrics = calculateMetrics(g, 5);
+
+  expect(metrics.totalPages).toBe(3);
+  expect(metrics.totalEdges).toBe(4);
+
+  // Check degrees on nodes directly
+  const nodeA = g.nodes.get('A');
+  expect(nodeA?.inLinks).toBe(1);
+  expect(nodeA?.outLinks).toBe(2);
+
+  const nodeC = g.nodes.get('C');
+  expect(nodeC?.inLinks).toBe(2);
+  expect(nodeC?.outLinks).toBe(1);
+
+  expect(metrics.averageOutDegree).toBeCloseTo(4/3);
+
+  // Top authority should be C with 2 in-links, authority = 1
+  expect(metrics.topAuthorityPages[0].url).toBe('C');
+  expect(metrics.topAuthorityPages[0].authority).toBeCloseTo(1);
+
+  // Max depth found
+  expect(metrics.maxDepthFound).toBe(1);
+
+  // Orphan pages (none)
+  expect(metrics.orphanPages).toEqual([]);
+});
+
+test('orphan pages', () => {
+  const g = new Graph();
+  g.addNode('Root', 0, 200);
+  g.addNode('Orphan', 1, 200);
+  // Orphan is at depth 1 but no incoming edges recorded (maybe missed or filtered)
+
+  const metrics = calculateMetrics(g, 5);
+  expect(metrics.orphanPages).toContain('Orphan');
+  expect(metrics.orphanPages).not.toContain('Root');
+});
+test('metrics v2 calculations', () => {
+  const g = new Graph();
+
+  // Root (depth 0, in=0, out=2)
+  g.addNode('root', 0, 200);
+
+  // A (depth 1, in=1, out=1)
+  g.addNode('A', 1, 200);
+  g.addEdge('root', 'A');
+
+  // B (depth 1, in=1, out=0)
+  g.addNode('B', 1, 200);
+  g.addEdge('root', 'B');
+
+  // C (depth 2, in=1, out=0)
+  g.addNode('C', 2, 200);
+  g.addEdge('A', 'C');
+
+  // Orphan (depth 1, in=0) - e.g. added but no edge to it?
+  // If it's in graph with depth > 0 and inLinks=0, it's an orphan.
+  g.addNode('orphan', 1, 200);
+
+  // Near Orphan (depth 3, in=1)
+  g.addNode('D', 2, 200);
+  g.addNode('nearOrphan', 3, 200);
+  g.addEdge('C', 'D'); // C->D
+  g.addEdge('D', 'nearOrphan'); // D->nearOrphan
+
+  // Deep page (depth 4)
+  g.addNode('deep', 4, 200);
+  g.addEdge('nearOrphan', 'deep');
+
+  // Nodes: root(0), A(1), B(1), C(2), orphan(1), D(2), nearOrphan(3), deep(4)
+  // Total pages: 8
+
+  // Edges: root->A, root->B, A->C, C->D, D->nearOrphan, nearOrphan->deep
+  // Total edges: 6
+
+  // InLinks:
+  // root: 0
+  // A: 1
+  // B: 1
+  // C: 1
+  // orphan: 0
+  // D: 1
+  // nearOrphan: 1
+  // deep: 1
+
+  // Max InLinks = 1.
+  // Authority Score = log(1 + in) / log(1 + maxIn)
+  // If maxIn = 1, log(2).
+  // For A: log(2)/log(2) = 1.
+  // For root: log(1)/log(2) = 0.
+
+  // Let's make maxIn > 1 to test better.
+  g.addNode('popular', 1, 200);
+  g.addEdge('root', 'popular');
+  g.addEdge('A', 'popular');
+  // popular inLinks = 2. MaxIn = 2.
+  // Authority popular = log(3)/log(3) = 1.
+  // Authority A = log(2)/log(3) approx 0.63
+
+  const metrics = calculateMetrics(g, 10); // maxDepth arg (not used for calculation logic of deepPages which is hardcoded >=4 per prompt?)
+  // Prompt says "deepPages: depth >= 4".
+  // Existing calculateMetrics takes maxDepth arg.
+  // Existing: deepPages = nodes.filter(n => n.depth >= maxDepth)
+  // New requirement: deepPages: depth >= 4.
+  // I should probably ignore the argument or update the requirement interpretation.
+  // "deepPages: depth >= 4" implies fixed threshold.
+
+  // Orphan pages: inLinks === 0 && depth > 0
+  expect(metrics.orphanPages).toContain('orphan');
+  expect(metrics.orphanPages).not.toContain('root'); // depth 0
+
+  // Near orphans: inLinks === 1 && depth >= 3
+  expect(metrics.nearOrphans).toContain('nearOrphan'); // depth 3, in 1
+  expect(metrics.nearOrphans).toContain('deep'); // depth 4, in 1 (from nearOrphan)
+  expect(metrics.nearOrphans).not.toContain('D'); // depth 2
+
+  // Deep pages: depth >= 4
+  expect(metrics.deepPages).toContain('deep');
+  expect(metrics.deepPages).not.toContain('nearOrphan');
+
+  // Crawl Efficiency Score: 1 - (deepPagesCount / totalPages)
+  // Total: 9 nodes (root, A, B, C, orphan, D, nearOrphan, deep, popular)
+  // Deep: 1 (deep)
+  // Score: 1 - 1/9 = 8/9 = 0.888...
+  expect(metrics.crawlEfficiencyScore).toBeCloseTo(8/9);
+
+  // Average Depth: sum(depth) / totalPages
+  // Depths: 0, 1, 1, 2, 1, 2, 3, 4, 1
+  // Sum: 15
+  // Avg: 15/9 = 1.666...
+  expect(metrics.averageDepth).toBeCloseTo(15/9);
+
+  // Structural Entropy
+  // OutDegrees:
+  // root: 3 (A, B, popular)
+  // A: 2 (C, popular)
+  // B: 0
+  // C: 1 (D)
+  // orphan: 0
+  // D: 1 (nearOrphan)
+  // nearOrphan: 1 (deep)
+  // deep: 0
+  // popular: 0
+
+  // Distribution:
+  // 0: 4 nodes (B, orphan, deep, popular)
+  // 1: 3 nodes (C, D, nearOrphan)
+  // 2: 1 node (A)
+  // 3: 1 node (root)
+
+  // P(0) = 4/9
+  // P(1) = 3/9
+  // P(2) = 1/9
+  // P(3) = 1/9
+
+  // Entropy = - (4/9 log2(4/9) + 3/9 log2(3/9) + 1/9 log2(1/9) + 1/9 log2(1/9))
+  // = - (0.444 * -1.17 + 0.333 * -1.58 + 0.111 * -3.17 + 0.111 * -3.17)
+  // approx 1.75
+
+  // Let's compute exact expected value
+  const p0 = 4/9;
+  const p1 = 3/9;
+  const p2 = 1/9;
+  const p3 = 1/9;
+  const entropy = - (p0 * Math.log2(p0) + p1 * Math.log2(p1) + p2 * Math.log2(p2) + p3 * Math.log2(p3));
+
+  expect(metrics.structuralEntropy).toBeCloseTo(entropy);
+
+  // Limit Reached
+  expect(metrics.limitReached).toBe(false);
+  g.limitReached = true;
+  const metrics2 = calculateMetrics(g, 10);
+  expect(metrics2.limitReached).toBe(true);
+});

package/tests/normalize.test.ts

@@ -0,0 +1,101 @@
+import { normalizeUrl } from '../src/crawler/normalize.js';
+import { extractLinks } from '../src/crawler/extract.js';
+import { test, expect } from 'vitest';
+
+test('normalizeUrl', () => {
+  expect(normalizeUrl('https://Example.com/Foo/', '')).toBe('https://example.com/Foo');
+  expect(normalizeUrl('http://example.com:80/bar', '')).toBe('http://example.com/bar');
+  expect(normalizeUrl('https://example.com/baz#frag', '')).toBe('https://example.com/baz');
+  expect(normalizeUrl('https://example.com/qux?a=1', '', { stripQuery: true })).toBe('https://example.com/qux');
+  expect(normalizeUrl('https://example.com/qux?a=1', '', { stripQuery: false })).toBe('https://example.com/qux?a=1');
+  expect(normalizeUrl('https://example.com/', '')).toBe('https://example.com/');
+});
+
+test('extractLinks', () => {
+  const html = `
+    <html>
+      <body>
+        <a href="/foo">Foo</a>
+        <a href="bar">Bar</a>
+        <a href="https://other.com/baz">Baz</a>
+        <a href="#top">Top</a>
+      </body>
+    </html>
+  `;
+  const links = extractLinks(html, 'https://example.com/page/');
+  expect(links).toContain('https://example.com/foo');
+  expect(links).toContain('https://example.com/page/bar');
+  expect(links).toContain('https://other.com/baz');
+  expect(links).not.toContain('https://example.com/page/#top');
+  expect(links).toContain('https://example.com/page/'); // #top resolves to base url without fragment
+});
+test('normalizeUrl: absolute resolution', () => {
+  expect(normalizeUrl('/foo', 'https://example.com')).toBe('https://example.com/foo');
+  expect(normalizeUrl('bar', 'https://example.com/baz/')).toBe('https://example.com/baz/bar');
+  expect(normalizeUrl('//other.com/foo', 'https://example.com')).toBe('https://other.com/foo');
+});
+
+test('normalizeUrl: only http/https', () => {
+  expect(normalizeUrl('ftp://example.com/file', 'https://example.com')).toBeNull();
+  expect(normalizeUrl('mailto:user@example.com', 'https://example.com')).toBeNull();
+  expect(normalizeUrl('javascript:alert(1)', 'https://example.com')).toBeNull();
+});
+
+test('normalizeUrl: lowercase hostname', () => {
+  expect(normalizeUrl('https://EXAMPLE.com/foo', '')).toBe('https://example.com/foo');
+});
+
+test('normalizeUrl: remove default ports', () => {
+  expect(normalizeUrl('http://example.com:80/foo', '')).toBe('http://example.com/foo');
+  expect(normalizeUrl('https://example.com:443/foo', '')).toBe('https://example.com/foo');
+  expect(normalizeUrl('http://example.com:8080/foo', '')).toBe('http://example.com:8080/foo');
+});
+
+test('normalizeUrl: remove hash fragments', () => {
+  expect(normalizeUrl('https://example.com/foo#bar', '')).toBe('https://example.com/foo');
+});
+
+test('normalizeUrl: strip query', () => {
+  expect(normalizeUrl('https://example.com/foo?a=1&b=2', '', { stripQuery: true })).toBe('https://example.com/foo');
+});
+
+test('normalizeUrl: filter tracking params', () => {
+  const url = 'https://example.com/foo?utm_source=google&utm_medium=cpc&a=1&fbclid=123';
+  expect(normalizeUrl(url, '', { stripQuery: false })).toBe('https://example.com/foo?a=1');
+
+  const url2 = 'https://example.com/foo?gclid=abc&msclkid=def';
+  expect(normalizeUrl(url2, '', { stripQuery: false })).toBe('https://example.com/foo');
+});
+
+test('normalizeUrl: trailing slash', () => {
+  expect(normalizeUrl('https://example.com/foo/', '')).toBe('https://example.com/foo');
+  expect(normalizeUrl('https://example.com/', '')).toBe('https://example.com/');
+});
+
+test('normalizeUrl: collapse duplicate slashes', () => {
+  expect(normalizeUrl('https://example.com/foo//bar', '')).toBe('https://example.com/foo/bar');
+  expect(normalizeUrl('https://example.com//foo///bar', '')).toBe('https://example.com/foo/bar');
+});
+
+test('normalizeUrl: skip non-HTML assets', () => {
+  expect(normalizeUrl('https://example.com/file.pdf', '')).toBeNull();
+  expect(normalizeUrl('https://example.com/image.jpg', '')).toBeNull();
+  expect(normalizeUrl('https://example.com/image.png', '')).toBeNull();
+  expect(normalizeUrl('https://example.com/image.svg', '')).toBeNull();
+  expect(normalizeUrl('https://example.com/image.webp', '')).toBeNull();
+  expect(normalizeUrl('https://example.com/image.gif', '')).toBeNull();
+  expect(normalizeUrl('https://example.com/archive.zip', '')).toBeNull();
+  expect(normalizeUrl('https://example.com/data.xml', '')).toBeNull();
+  expect(normalizeUrl('https://example.com/data.json', '')).toBeNull();
+  expect(normalizeUrl('https://example.com/video.mp4', '')).toBeNull();
+
+  // HTML extensions should pass (or no extension)
+  expect(normalizeUrl('https://example.com/page.html', '')).toBe('https://example.com/page.html');
+  expect(normalizeUrl('https://example.com/page.htm', '')).toBe('https://example.com/page.htm');
+  expect(normalizeUrl('https://example.com/page', '')).toBe('https://example.com/page');
+});
+
+test('normalizeUrl: return format', () => {
+  const res = normalizeUrl('https://example.com/foo?a=1', '');
+  expect(res).toBe('https://example.com/foo?a=1');
+});

package/tests/orphanSeverity.test.ts

@@ -0,0 +1,160 @@
+import { describe, expect, test } from 'vitest';
+import { annotateOrphans, calculateOrphanSeverity, mapImpactLevel, type SitegraphNode, type SitegraphEdge } from '../src/scoring/orphanSeverity.js';
+
+function baseNode(url: string, overrides: Partial<SitegraphNode> = {}): SitegraphNode {
+  return {
+    url,
+    depth: 1,
+    inLinks: 0,
+    outLinks: 0,
+    status: 200,
+    ...overrides
+  };
+}
+
+describe('orphan detection and severity scoring', () => {
+  test('hard orphan detection and homepage exclusion', () => {
+    const nodes: SitegraphNode[] = [
+      baseNode('https://example.com/', { depth: 0, inLinks: 0 }),
+      baseNode('https://example.com/orphan', { inLinks: 0 })
+    ];
+    const edges: SitegraphEdge[] = [];
+
+    const result = annotateOrphans(nodes, edges, {
+      enabled: true,
+      severityEnabled: false,
+      includeSoftOrphans: false,
+      minInbound: 2,
+      rootUrl: 'https://example.com/'
+    });
+
+    expect(result[0]).toMatchObject({ orphan: false });
+    expect(result[1]).toMatchObject({ orphan: true, orphanType: 'hard' });
+  });
+
+  test('near orphan threshold override', () => {
+    const nodes = [baseNode('https://example.com/near', { inLinks: 2 })];
+    const edges: SitegraphEdge[] = [];
+
+    const resultDefault = annotateOrphans(nodes, edges, {
+      enabled: true,
+      severityEnabled: false,
+      includeSoftOrphans: false,
+      minInbound: 2
+    });
+    const resultStrict = annotateOrphans(nodes, edges, {
+      enabled: true,
+      severityEnabled: false,
+      includeSoftOrphans: false,
+      minInbound: 1
+    });
+
+    expect(resultDefault[0]).toMatchObject({ orphan: true, orphanType: 'near' });
+    expect(resultStrict[0]).toMatchObject({ orphan: false });
+  });
+
+  test('soft orphan detection only when enabled and inbound only from low-value sources', () => {
+    const nodes: SitegraphNode[] = [
+      baseNode('https://example.com/tag/seo', { pageType: 'tag', outLinks: 1 }),
+      baseNode('https://example.com/list?page=2', { pageType: 'pagination', outLinks: 1 }),
+      baseNode('https://example.com/target', { inLinks: 2 }),
+      baseNode('https://example.com/normal', { outLinks: 1 })
+    ];
+
+    const edges: SitegraphEdge[] = [
+      { source: 'https://example.com/tag/seo', target: 'https://example.com/target' },
+      { source: 'https://example.com/list?page=2', target: 'https://example.com/target' }
+    ];
+
+    const withSoft = annotateOrphans(nodes, edges, {
+      enabled: true,
+      severityEnabled: false,
+      includeSoftOrphans: true,
+      minInbound: 1
+    });
+
+    const withoutSoft = annotateOrphans(nodes, edges, {
+      enabled: true,
+      severityEnabled: false,
+      includeSoftOrphans: false,
+      minInbound: 1
+    });
+
+    expect(withSoft.find((n) => n.url.endsWith('/target'))).toMatchObject({ orphan: true, orphanType: 'soft' });
+    expect(withoutSoft.find((n) => n.url.endsWith('/target'))).toMatchObject({ orphan: false });
+  });
+
+  test('crawl-only orphan detection', () => {
+    const nodes = [baseNode('https://example.com/sitemap-only', { inLinks: 0, discoveredViaSitemap: true })];
+    const result = annotateOrphans(nodes, [], {
+      enabled: true,
+      severityEnabled: false,
+      includeSoftOrphans: false,
+      minInbound: 2
+    });
+
+    expect(result[0]).toMatchObject({ orphan: true, orphanType: 'crawl-only' });
+  });
+
+  test('severity calculation modifiers and score clamping', () => {
+    const high = calculateOrphanSeverity('hard', baseNode('https://example.com/high', {
+      inLinks: 0,
+      wordCount: 1500,
+      hasStructuredData: true,
+      depth: 1,
+      isProductOrCommercial: true
+    }));
+
+    const low = calculateOrphanSeverity('hard', baseNode('https://example.com/low', {
+      inLinks: 0,
+      wordCount: 120,
+      noindex: true,
+      duplicateContent: true,
+      pageType: 'archive'
+    }));
+
+    expect(high).toBe(100);
+    expect(low).toBe(80);
+  });
+
+  test('impact level mapping', () => {
+    expect(mapImpactLevel(0)).toBe('low');
+    expect(mapImpactLevel(39)).toBe('low');
+    expect(mapImpactLevel(40)).toBe('medium');
+    expect(mapImpactLevel(69)).toBe('medium');
+    expect(mapImpactLevel(70)).toBe('high');
+    expect(mapImpactLevel(89)).toBe('high');
+    expect(mapImpactLevel(90)).toBe('critical');
+    expect(mapImpactLevel(100)).toBe('critical');
+  });
+
+  test('canonical consolidation, robots exclusion, and deterministic JSON output snapshot', () => {
+    const nodes: SitegraphNode[] = [
+      baseNode('https://example.com/canonical', { inLinks: 0 }),
+      baseNode('https://example.com/variant?a=1', { canonicalUrl: 'https://example.com/canonical', inLinks: 1 }),
+      baseNode('https://example.com/blocked', { inLinks: 0, robotsExcluded: true }),
+      baseNode('https://example.com/redirect-target', { inLinks: 1 })
+    ];
+
+    const edges: SitegraphEdge[] = [
+      { source: 'https://example.com/redirect-source', target: 'https://example.com/redirect-target' }
+    ];
+
+    const options = {
+      enabled: true,
+      severityEnabled: true,
+      includeSoftOrphans: true,
+      minInbound: 2
+    };
+
+    const first = annotateOrphans(nodes, edges, options);
+    const second = annotateOrphans(nodes, edges, options);
+
+    expect(first).toEqual(second);
+    expect(first.find((n) => n.url.endsWith('/canonical'))).toMatchObject({ orphan: true, orphanType: 'near' });
+    expect(first.find((n) => n.url.endsWith('/blocked'))).toMatchObject({ orphan: false });
+
+    const normalized = JSON.stringify(first, null, 2).replace(/\r\n/g, '\n');
+    expect(normalized).toMatchSnapshot();
+  });
+});

package/tests/pagerank.test.ts

@@ -0,0 +1,98 @@
+import { describe, it, expect } from 'vitest';
+import { Graph } from '../src/graph/graph.js';
+import { computePageRank } from '../src/graph/pagerank.js';
+
+describe('PageRank Engine', () => {
+    it('should calculate identical PageRank for a simple loop', () => {
+        const graph = new Graph();
+        graph.addNode('https://a.com', 0, 200);
+        graph.addNode('https://b.com', 1, 200);
+        graph.addEdge('https://a.com', 'https://b.com');
+        graph.addEdge('https://b.com', 'https://a.com');
+
+        computePageRank(graph);
+        const nodes = graph.getNodes();
+
+        expect(nodes[0].pageRank).toBeCloseTo(0.5, 4);
+        expect(nodes[1].pageRank).toBeCloseTo(0.5, 4);
+        expect(nodes[0].pageRankScore).toBe(100);
+        expect(nodes[1].pageRankScore).toBe(100);
+    });
+
+    it('should identify the center of a star graph as most important', () => {
+        const graph = new Graph();
+        graph.addNode('https://center.com', 0, 200);
+        graph.addNode('https://p1.com', 1, 200);
+        graph.addNode('https://p2.com', 1, 200);
+        graph.addNode('https://p3.com', 1, 200);
+
+        // Star in: all link to center
+        graph.addEdge('https://p1.com', 'https://center.com');
+        graph.addEdge('https://p2.com', 'https://center.com');
+        graph.addEdge('https://p3.com', 'https://center.com');
+
+        computePageRank(graph);
+        const nodes = graph.getNodes();
+
+        const center = nodes.find(n => n.url.includes('center'))!;
+        const leaves = nodes.filter(n => !n.url.includes('center'));
+
+        expect(center.pageRankScore).toBe(100);
+        leaves.forEach(leaf => {
+            expect(leaf.pageRankScore).toBeLessThan(100);
+            expect(leaf.pageRank!).toBeLessThan(center.pageRank!);
+        });
+    });
+
+    it('should respect link weights (Body > Nav > Footer)', () => {
+        const graph = new Graph();
+        graph.addNode('https://source.com', 0, 200);
+        graph.addNode('https://body-target.com', 1, 200);
+        graph.addNode('https://footer-target.com', 1, 200);
+
+        // Body weight 1.0, Footer weight 0.4
+        graph.addEdge('https://source.com', 'https://body-target.com', 1.0);
+        graph.addEdge('https://source.com', 'https://footer-target.com', 0.4);
+
+        computePageRank(graph);
+
+        const bodyTarget = graph.nodes.get('https://body-target.com')!;
+        const footerTarget = graph.nodes.get('https://footer-target.com')!;
+
+        expect(bodyTarget.pageRank!).toBeGreaterThan(footerTarget.pageRank!);
+    });
+
+    it('should handle sink nodes by redistributing rank', () => {
+        const graph = new Graph();
+        graph.addNode('https://a.com', 0, 200);
+        graph.addNode('https://b.com', 1, 200); // b is a sink
+        graph.addEdge('https://a.com', 'https://b.com');
+
+        computePageRank(graph);
+
+        const nodeA = graph.nodes.get('https://a.com')!;
+        const nodeB = graph.nodes.get('https://b.com')!;
+
+        // Without redistribution, A would lose all rank.
+        // With redistribution, A should still have some rank.
+        expect(nodeA.pageRank).toBeGreaterThan(0);
+        expect(nodeB.pageRank).toBeGreaterThan(nodeA.pageRank!);
+    });
+
+    it('should exclude noindex pages from receiving or passing rank', () => {
+        const graph = new Graph();
+        graph.addNode('https://a.com', 0, 200);
+        graph.addNode('https://no-index.com', 1, 200);
+        graph.nodes.get('https://no-index.com')!.noindex = true;
+
+        graph.addEdge('https://a.com', 'https://no-index.com');
+
+        computePageRank(graph);
+
+        const nodeA = graph.nodes.get('https://a.com')!;
+        const nodeNoIndex = graph.nodes.get('https://no-index.com')!;
+
+        expect(nodeNoIndex.pageRank).toBeUndefined();
+        expect(nodeA.pageRank).toBe(1.0); // Only one eligible node
+    });
+});

package/tests/parser.test.ts

@@ -0,0 +1,117 @@
+import { test, expect } from 'vitest';
+import { Parser } from '../src/crawler/parser.js';
+
+const parser = new Parser();
+const baseUrl = 'https://example.com';
+
+test('extracts links correctly', () => {
+  const html = `
+    <html>
+      <body>
+        <a href="/page1">Page 1</a>
+        <a href="https://other.com">Other</a>
+        <a href="#hash">Hash</a>
+        <a href="javascript:void(0)">JS</a>
+      </body>
+    </html>
+  `;
+  const result = parser.parse(html, baseUrl, 200);
+  const urls = result.links.map(l => l.url);
+  expect(urls).toContain('https://example.com/page1');
+  expect(urls).toContain('https://other.com/');
+  expect(urls).not.toContain('https://example.com/#hash');
+  // It also extracts the base URL itself from href="#hash"
+  expect(urls).toContain('https://example.com/');
+  expect(result.links.length).toBe(3);
+});
+
+test('respects nofollow on links', () => {
+  const html = `
+    <html>
+      <body>
+        <a href="/page1" rel="nofollow">Page 1</a>
+        <a href="/page2">Page 2</a>
+      </body>
+    </html>
+  `;
+  const result = parser.parse(html, baseUrl, 200);
+  const urls = result.links.map(l => l.url);
+  expect(urls).not.toContain('https://example.com/page1');
+  expect(urls).toContain('https://example.com/page2');
+});
+
+test('respects meta robots nofollow', () => {
+  const html = `
+    <html>
+      <head>
+        <meta name="robots" content="nofollow">
+      </head>
+      <body>
+        <a href="/page1">Page 1</a>
+      </body>
+    </html>
+  `;
+  const result = parser.parse(html, baseUrl, 200);
+  expect(result.nofollow).toBe(true);
+  expect(result.links.length).toBe(0);
+});
+
+test('detects canonical', () => {
+  const html = `
+    <html>
+      <head>
+        <link rel="canonical" href="https://example.com/canon">
+      </head>
+    </html>
+  `;
+  const result = parser.parse(html, baseUrl, 200);
+  expect(result.canonical).toBe('https://example.com/canon');
+});
+
+test('detects relative canonical', () => {
+  const html = `
+    <html>
+      <head>
+        <link rel="canonical" href="/canon">
+      </head>
+    </html>
+  `;
+  const result = parser.parse(html, baseUrl, 200);
+  expect(result.canonical).toBe('https://example.com/canon');
+});
+
+test('detects soft 404', () => {
+  const html = `
+    <html>
+      <head><title>Page Not Found</title></head>
+      <body>Sorry, the page you are looking for does not exist.</body>
+    </html>
+  `;
+  const result = parser.parse(html, baseUrl, 200);
+  expect(result.soft404Score).toBeGreaterThanOrEqual(0.5);
+});
+
+test('content hash ignores scripts', () => {
+  const html1 = `
+    <html><body><script>var x=1;</script><p>Hello</p></body></html>
+  `;
+  const html2 = `
+    <html><body><script>var x=2;</script><p>Hello</p></body></html>
+  `;
+  const result1 = parser.parse(html1, baseUrl, 200);
+  const result2 = parser.parse(html2, baseUrl, 200);
+  expect(result1.contentHash).toBe(result2.contentHash);
+});
+
+test('detects meta robots noindex', () => {
+  const html = `
+    <html>
+      <head>
+        <meta name="robots" content="noindex, nofollow">
+      </head>
+    </html>
+  `;
+  const result = parser.parse(html, baseUrl, 200);
+  expect(result.noindex).toBe(true);
+  expect(result.nofollow).toBe(true);
+});