@crawlith/core 0.1.0 → 0.1.1

package/tests/orphanSeverity.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, test } from 'vitest';
-import { annotateOrphans, calculateOrphanSeverity, mapImpactLevel, type SitegraphNode, type SitegraphEdge } from '../src/scoring/orphanSeverity.js';
+import { annotateOrphans, calculateOrphanSeverity, mapImpactLevel, type CrawlNode, type CrawlEdge } from '../src/scoring/orphanSeverity.js';
 
-function baseNode(url: string, overrides: Partial<SitegraphNode> = {}): SitegraphNode {
+function baseNode(url: string, overrides: Partial<CrawlNode> = {}): CrawlNode {
   return {
     url,
     depth: 1,
@@ -14,11 +14,11 @@ function baseNode(url: string, overrides: Partial<SitegraphNode> = {}): Sitegrap
 
 describe('orphan detection and severity scoring', () => {
   test('hard orphan detection and homepage exclusion', () => {
-    const nodes: SitegraphNode[] = [
+    const nodes: CrawlNode[] = [
       baseNode('https://example.com/', { depth: 0, inLinks: 0 }),
       baseNode('https://example.com/orphan', { inLinks: 0 })
     ];
-    const edges: SitegraphEdge[] = [];
+    const edges: CrawlEdge[] = [];
 
     const result = annotateOrphans(nodes, edges, {
       enabled: true,
@@ -34,7 +34,7 @@ describe('orphan detection and severity scoring', () => {
 
   test('near orphan threshold override', () => {
     const nodes = [baseNode('https://example.com/near', { inLinks: 2 })];
-    const edges: SitegraphEdge[] = [];
+    const edges: CrawlEdge[] = [];
 
     const resultDefault = annotateOrphans(nodes, edges, {
       enabled: true,
@@ -54,14 +54,14 @@ describe('orphan detection and severity scoring', () => {
   });
 
   test('soft orphan detection only when enabled and inbound only from low-value sources', () => {
-    const nodes: SitegraphNode[] = [
+    const nodes: CrawlNode[] = [
       baseNode('https://example.com/tag/seo', { pageType: 'tag', outLinks: 1 }),
       baseNode('https://example.com/list?page=2', { pageType: 'pagination', outLinks: 1 }),
       baseNode('https://example.com/target', { inLinks: 2 }),
       baseNode('https://example.com/normal', { outLinks: 1 })
     ];
 
-    const edges: SitegraphEdge[] = [
+    const edges: CrawlEdge[] = [
       { source: 'https://example.com/tag/seo', target: 'https://example.com/target' },
       { source: 'https://example.com/list?page=2', target: 'https://example.com/target' }
     ];
@@ -129,14 +129,14 @@ describe('orphan detection and severity scoring', () => {
   });
 
   test('canonical consolidation, robots exclusion, and deterministic JSON output snapshot', () => {
-    const nodes: SitegraphNode[] = [
+    const nodes: CrawlNode[] = [
       baseNode('https://example.com/canonical', { inLinks: 0 }),
       baseNode('https://example.com/variant?a=1', { canonicalUrl: 'https://example.com/canonical', inLinks: 1 }),
       baseNode('https://example.com/blocked', { inLinks: 0, robotsExcluded: true }),
       baseNode('https://example.com/redirect-target', { inLinks: 1 })
     ];
 
-    const edges: SitegraphEdge[] = [
+    const edges: CrawlEdge[] = [
       { source: 'https://example.com/redirect-source', target: 'https://example.com/redirect-target' }
     ];
 

package/tests/redirect_safety.test.ts

@@ -5,7 +5,11 @@ import { request } from 'undici';
 
 vi.mock('undici', () => ({
     request: vi.fn(),
-    ProxyAgent: vi.fn().mockImplementation(() => ({ dispatcher: {} }))
+    ProxyAgent: vi.fn().mockImplementation(() => ({ dispatcher: {} })),
+    Agent: class {
+        dispatch = vi.fn();
+    },
+    Dispatcher: class {}
 }));
 
 describe('RedirectController', () => {

package/tests/renderAnalysisCsv.test.ts

@@ -0,0 +1,183 @@
+import { describe, expect, test } from 'vitest';
+import { renderAnalysisCsv, AnalysisResult } from '../src/analysis/analyze.js';
+
+describe('renderAnalysisCsv', () => {
+    test('renders CSV with headers', () => {
+        const result: AnalysisResult = {
+            pages: [],
+            site_summary: {
+                pages_analyzed: 0,
+                avg_seo_score: 0,
+                thin_pages: 0,
+                duplicate_titles: 0,
+                site_score: 0
+            },
+            site_scores: {} as any,
+            active_modules: {
+                seo: true,
+                content: true,
+                accessibility: true
+            }
+        };
+
+        const csv = renderAnalysisCsv(result);
+        expect(csv).toContain('URL,SEO Score,Thin Score,HTTP Status,Title,Title Length,Meta Description,Desc Length,Word Count,Internal Links,External Links');
+    });
+
+    test('renders a single page correctly', () => {
+        const result: AnalysisResult = {
+            pages: [
+                {
+                    url: 'https://example.com',
+                    status: 200,
+                    seoScore: 85,
+                    thinScore: 10,
+                    title: { value: 'Example Domain', length: 14, status: 'ok' },
+                    metaDescription: { value: 'This is an example description.', length: 29, status: 'ok' },
+                    content: { wordCount: 500 } as any,
+                    links: { internalLinks: 5, externalLinks: 2 } as any,
+                    h1: {} as any,
+                    images: {} as any,
+                    structuredData: {} as any,
+                    meta: {}
+                }
+            ],
+            site_summary: {
+                pages_analyzed: 1,
+                avg_seo_score: 85,
+                thin_pages: 0,
+                duplicate_titles: 0,
+                site_score: 85
+            },
+            site_scores: {} as any,
+            active_modules: {
+                seo: true,
+                content: true,
+                accessibility: true
+            }
+        };
+
+        const csv = renderAnalysisCsv(result);
+        const lines = csv.split('\n');
+        expect(lines.length).toBe(2);
+        expect(lines[1]).toContain('https://example.com,85,10,200,"Example Domain",14,"This is an example description.",29,500,5,2');
+    });
+
+    test('escapes quotes in title and meta description', () => {
+        const result: AnalysisResult = {
+            pages: [
+                {
+                    url: 'https://example.com/quote',
+                    status: 200,
+                    seoScore: 90,
+                    thinScore: 5,
+                    title: { value: 'Example "Quoted" Domain', length: 23, status: 'ok' },
+                    metaDescription: { value: 'This description contains "quotes" inside.', length: 42, status: 'ok' },
+                    content: { wordCount: 300 } as any,
+                    links: { internalLinks: 3, externalLinks: 1 } as any,
+                    h1: {} as any,
+                    images: {} as any,
+                    structuredData: {} as any,
+                    meta: {}
+                }
+            ],
+            site_summary: {
+                pages_analyzed: 1,
+                avg_seo_score: 90,
+                thin_pages: 0,
+                duplicate_titles: 0,
+                site_score: 90
+            },
+            site_scores: {} as any,
+            active_modules: {
+                seo: true,
+                content: true,
+                accessibility: true
+            }
+        };
+
+        const csv = renderAnalysisCsv(result);
+        const lines = csv.split('\n');
+        // Expect double quotes to be escaped with double quotes: " -> ""
+        // And the whole field wrapped in quotes
+        expect(lines[1]).toContain('"Example ""Quoted"" Domain"');
+        expect(lines[1]).toContain('"This description contains ""quotes"" inside."');
+    });
+
+    test('handles Pending/Limit status (status: 0)', () => {
+        const result: AnalysisResult = {
+            pages: [
+                {
+                    url: 'https://example.com/pending',
+                    status: 0,
+                    seoScore: 0,
+                    thinScore: 0,
+                    title: { value: null, length: 0, status: 'missing' },
+                    metaDescription: { value: null, length: 0, status: 'missing' },
+                    content: { wordCount: 0 } as any,
+                    links: { internalLinks: 0, externalLinks: 0 } as any,
+                    h1: {} as any,
+                    images: {} as any,
+                    structuredData: {} as any,
+                    meta: {}
+                }
+            ],
+            site_summary: {
+                pages_analyzed: 1,
+                avg_seo_score: 0,
+                thin_pages: 0,
+                duplicate_titles: 0,
+                site_score: 0
+            },
+            site_scores: {} as any,
+            active_modules: {
+                seo: true,
+                content: true,
+                accessibility: true
+            }
+        };
+
+        const csv = renderAnalysisCsv(result);
+        const lines = csv.split('\n');
+        expect(lines[1]).toContain('Pending/Limit');
+    });
+
+    test('handles missing title and description gracefully', () => {
+        const result: AnalysisResult = {
+            pages: [
+                {
+                    url: 'https://example.com/missing',
+                    status: 404,
+                    seoScore: 0,
+                    thinScore: 0,
+                    title: { value: undefined as any, length: 0, status: 'missing' },
+                    metaDescription: { value: null as any, length: 0, status: 'missing' },
+                    content: { wordCount: 0 } as any,
+                    links: { internalLinks: 0, externalLinks: 0 } as any,
+                    h1: {} as any,
+                    images: {} as any,
+                    structuredData: {} as any,
+                    meta: {}
+                }
+            ],
+            site_summary: {
+                pages_analyzed: 1,
+                avg_seo_score: 0,
+                thin_pages: 0,
+                duplicate_titles: 0,
+                site_score: 0
+            },
+            site_scores: {} as any,
+            active_modules: {
+                seo: true,
+                content: true,
+                accessibility: true
+            }
+        };
+
+        const csv = renderAnalysisCsv(result);
+        const lines = csv.split('\n');
+        // Should produce empty quoted strings ""
+        expect(lines[1]).toContain(',"",0,"",0,0,0,0');
+    });
+});

package/tests/safety.test.ts

@@ -34,6 +34,18 @@ describe('IPGuard', () => {
         expect(IPGuard.isInternal('fe80::1')).toBe(true);
     });
 
+    it('should block IPv4-mapped IPv6 internal addresses', () => {
+        expect(IPGuard.isInternal('::ffff:127.0.0.1')).toBe(true);
+        expect(IPGuard.isInternal('::ffff:10.0.0.1')).toBe(true);
+        expect(IPGuard.isInternal('::ffff:192.168.1.1')).toBe(true);
+        expect(IPGuard.isInternal('::ffff:169.254.169.254')).toBe(true);
+        expect(IPGuard.isInternal('::ffff:7f00:0001')).toBe(true); // Hex 127.0.0.1
+    });
+
+    it('should allow IPv4-mapped IPv6 public addresses', () => {
+        expect(IPGuard.isInternal('::ffff:8.8.8.8')).toBe(false);
+    });
+
     it('should validate hostname by resolving IPs', async () => {
         const resolve4Spy = vi.mocked(dns.resolve4);
         const resolve6Spy = vi.mocked(dns.resolve6);

package/tests/scope.test.ts

@@ -25,6 +25,13 @@ describe('DomainFilter', () => {
         const filter = new DomainFilter(['allowed.com']);
         expect(filter.isAllowed('other.com')).toBe(false);
     });
+
+    it('should fallback to raw string on invalid hostname', () => {
+        // '[' and 'http://denied-invalid-[' causes new URL() to throw
+        const filter = new DomainFilter(['['], ['denied-invalid-[']);
+        expect(filter.isAllowed('[')).toBe(true);
+        expect(filter.isAllowed('denied-invalid-[')).toBe(false);
+    });
 });
 
 describe('SubdomainPolicy', () => {
@@ -63,4 +70,15 @@ describe('ScopeManager', () => {
         expect(manager.isUrlEligible('https://other.com/')).toBe('allowed');
         expect(manager.isUrlEligible('https://google.com/')).toBe('blocked_by_domain_filter');
     });
+
+    it('should handle trailing dots in hostnames', () => {
+        const manager = new ScopeManager({
+            rootUrl: 'https://example.com',
+            allowedDomains: ['example.com.'],
+            includeSubdomains: false
+        });
+
+        expect(manager.isUrlEligible('https://example.com./')).toBe('allowed');
+        expect(manager.isUrlEligible('https://example.com/')).toBe('allowed');
+    });
 });

package/tests/scoring.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from 'vitest';
-import { scorePageSeo, aggregateSiteScore } from '../src/analysis/scoring.js';
+import { scorePageSeo } from '../src/analysis/scoring.js';
 import { PageAnalysis } from '../src/analysis/analyze.js';
 
 const basePage: PageAnalysis = {
@@ -13,7 +13,8 @@ const basePage: PageAnalysis = {
   images: { totalImages: 2, missingAlt: 0, emptyAlt: 0 },
   links: { internalLinks: 5, externalLinks: 2, nofollowCount: 1, externalRatio: 2 / 7 },
   structuredData: { present: true, valid: true, types: ['Article'] },
-  seoScore: 0
+  seoScore: 0,
+  meta: { noindex: false, nofollow: false }
 };
 
 test('page score stays in 0-100', () => {
@@ -34,26 +35,26 @@ test('page score stays in 0-100', () => {
   expect(scorePageSeo(badPage)).toBeLessThan(50);
 });
 
-test('aggregate site score includes existing metrics signals', () => {
-  const score = aggregateSiteScore({
-    totalPages: 2,
-    totalEdges: 1,
-    orphanPages: ['https://example.com/x'],
-    nearOrphans: [],
-    deepPages: [],
-    topAuthorityPages: [{ url: 'a', authority: 1 }],
-    averageOutDegree: 1,
-    maxDepthFound: 1,
-    crawlEfficiencyScore: 0.8,
-    averageDepth: 1,
-    structuralEntropy: 2,
-    limitReached: false
-  }, [
-    { ...basePage, seoScore: 70 },
-    { ...basePage, seoScore: 90, url: 'https://example.com/2' }
-  ]);
+// test('aggregate site score includes existing metrics signals', () => {
+//   const score = aggregateSiteScore({
+//     totalPages: 2,
+//     totalEdges: 1,
+//     orphanPages: ['https://example.com/x'],
+//     nearOrphans: [],
+//     deepPages: [],
+//     topAuthorityPages: [{ url: 'a', authority: 1 }],
+//     averageOutDegree: 1,
+//     maxDepthFound: 1,
+//     crawlEfficiencyScore: 0.8,
+//     averageDepth: 1,
+//     structuralEntropy: 2,
+//     limitReached: false
+//   }, [
+//     { ...basePage, seoScore: 70 },
+//     { ...basePage, seoScore: 90, url: 'https://example.com/2' }
+//   ]);
 
-  expect(score.seoHealthScore).toBe(80);
-  expect(score.overallScore).toBeGreaterThan(0);
-  expect(score.overallScore).toBeLessThanOrEqual(100);
-});
+//   expect(score.seoHealthScore).toBe(80);
+//   expect(score.overallScore).toBeGreaterThan(0);
+//   expect(score.overallScore).toBeLessThanOrEqual(100);
+// });

package/tests/sitemap.test.ts

@@ -1,6 +1,7 @@
-import { test, expect, beforeEach } from 'vitest';
+import { test, expect, beforeEach, vi } from 'vitest';
 import { Sitemap } from '../src/crawler/sitemap.js';
 import { MockAgent, setGlobalDispatcher } from 'undici';
+import { EngineContext } from '../src/events.js';
 
 let mockAgent: MockAgent;
 
@@ -86,3 +87,14 @@ test('handles fetch errors gracefully', async () => {
   const urls = await sitemap.fetch('https://example.com/error.xml');
   expect(urls.length).toBe(0);
 });
+
+test('emits warning on fetch error', async () => {
+  const client = mockAgent.get('https://example.com');
+  client.intercept({ path: '/error.xml', method: 'GET' }).replyWithError(new Error('Network error'));
+
+  const mockContext: EngineContext = { emit: vi.fn() };
+  const sitemap = new Sitemap(mockContext);
+  await sitemap.fetch('https://example.com/error.xml');
+
+  expect(mockContext.emit).toHaveBeenCalledWith(expect.objectContaining({ type: 'warn' }));
+});

package/tests/ssrf_fix.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { Fetcher } from '../src/crawler/fetcher.js';
+import { request } from 'undici';
+import { IPGuard } from '../src/core/security/ipGuard.js';
+
+// Mock undici request to fail with EBLOCKED
+vi.mock('undici', () => {
+    return {
+        request: vi.fn(),
+        Agent: class {
+            dispatch = vi.fn();
+        },
+        Dispatcher: class {}
+    };
+});
+
+// Mock IPGuard.validateHost to pass
+vi.mock('../src/core/security/ipGuard.js', async () => {
+    const original = await vi.importActual('../src/core/security/ipGuard.js');
+    return {
+        ...original as any,
+        IPGuard: {
+            ...original.IPGuard,
+            validateHost: vi.fn().mockResolvedValue(true), // Pass step 1
+            getSecureDispatcher: vi.fn()
+        }
+    };
+});
+
+describe('SSRF Fix Reproduction', () => {
+    let fetcher: Fetcher;
+
+    beforeEach(() => {
+        vi.clearAllMocks();
+        // Setup default mock return for dispatcher
+        vi.mocked(IPGuard.getSecureDispatcher).mockReturnValue({} as any);
+        fetcher = new Fetcher({ rate: 100 });
+    });
+
+    it('should return blocked_internal_ip when secureDispatcher blocks', async () => {
+        const mockRequest = vi.mocked(request);
+        const mockGetSecureDispatcher = vi.mocked(IPGuard.getSecureDispatcher);
+        const mockDispatcher = { dispatch: vi.fn() } as any;
+        mockGetSecureDispatcher.mockReturnValue(mockDispatcher);
+
+        // Re-initialize fetcher so it calls getSecureDispatcher and gets our specific mock
+        fetcher = new Fetcher({ rate: 100 });
+
+        // Simulate secureDispatcher blocking via undici request throwing EBLOCKED
+        const blockedError = new Error('Blocked internal IP: 127.0.0.1');
+        (blockedError as any).code = 'EBLOCKED';
+
+        mockRequest.mockRejectedValueOnce(blockedError);
+
+        const res = await fetcher.fetch('http://example.com');
+
+        // Now we expect correct handling
+        expect(res.status).toBe('blocked_internal_ip');
+
+        // Verify that the secure dispatcher was indeed used
+        expect(mockGetSecureDispatcher).toHaveBeenCalled();
+        expect(mockRequest).toHaveBeenCalledWith(
+            expect.stringContaining('http://example.com'),
+            expect.objectContaining({
+                dispatcher: mockDispatcher
+            })
+        );
+    });
+});

package/tests/visualization_data.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import { SITEGRAPH_HTML } from '../src/report/sitegraph_template.js';
+import { Crawl_HTML } from '../src/report/crawl_template.js';
 import { Graph } from '../src/graph/graph.js';
 import { computePageRank } from '../src/graph/pagerank.js';
 
@@ -23,24 +23,24 @@ describe('Visualization Data & Template', () => {
     });
 
     it('should contain UI toggle buttons for Authority Mode', () => {
-        expect(SITEGRAPH_HTML).toContain('id="btn-auth-pagerank"');
-        expect(SITEGRAPH_HTML).toContain('id="btn-auth-structural"');
+        expect(Crawl_HTML).toContain('id="btn-auth-pagerank"');
+        expect(Crawl_HTML).toContain('id="btn-auth-structural"');
     });
 
     it('should contain setAuthorityMode function', () => {
         // Use regex to be flexible with whitespace
-        expect(SITEGRAPH_HTML).toMatch(/function\s+setAuthorityMode\s*\(mode,\s*btn\)/);
-        expect(SITEGRAPH_HTML).toContain('n.authority = mode === \'pagerank\' ? n.pageRankAuthority : n.structuralAuthority');
+        expect(Crawl_HTML).toMatch(/function\s+setAuthorityMode\s*\(mode,\s*btn\)/);
+        expect(Crawl_HTML).toContain('n.authority = mode === \'pagerank\' ? n.pageRankAuthority : n.structuralAuthority');
     });
 
     it('should contain logic to calculate pageRankAuthority from pageRankScore', () => {
-        expect(SITEGRAPH_HTML).toContain('n.pageRankAuthority = n.pageRankScore / 100');
-        expect(SITEGRAPH_HTML).toContain('n.structuralAuthority = Math.log(1 + n.inLinks)');
+        expect(Crawl_HTML).toContain('n.pageRankAuthority = n.pageRankScore / 100');
+        expect(Crawl_HTML).toContain('n.structuralAuthority = Math.log(1 + n.inLinks)');
     });
 
     it('should update details panel to show both metrics', () => {
-        expect(SITEGRAPH_HTML).toContain('id="d-auth-container"');
-        expect(SITEGRAPH_HTML).toContain('In-Degree: ${structVal}');
-        expect(SITEGRAPH_HTML).toContain('PR: <strong>${prVal}</strong>');
+        expect(Crawl_HTML).toContain('id="d-auth-container"');
+        expect(Crawl_HTML).toContain('In-Degree: ${structVal}');
+        expect(Crawl_HTML).toContain('PR: <strong>${prVal}</strong>');
     });
 });

package/dist/report/sitegraphExport.d.ts

@@ -1,3 +0,0 @@
-export declare function renderSitegraphCsvNodes(graphData: any): string;
-export declare function renderSitegraphCsvEdges(graphData: any): string;
-export declare function renderSitegraphMarkdown(url: string, graphData: any, metrics: any, graph: any): string;

package/dist/report/sitegraph_template.d.ts

@@ -1 +0,0 @@
-export declare const SITEGRAPH_HTML = "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n  <meta charset=\"UTF-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n  <title>Crawlith Site Graph</title>\n  <style>\n    :root {\n      --bg-color: #121212;\n      --text-color: #e0e0e0;\n      --panel-bg: #1e1e1e;\n      --border-color: #333;\n      --accent-color: #4a90e2;\n      --sidebar-width: 300px;\n    }\n    body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, Helvetica, Arial, sans-serif; background: var(--bg-color); color: var(--text-color); height: 100vh; display: flex; flex-direction: column; overflow: hidden; }\n\n    /* Layout */\n    header { padding: 0 20px; background: var(--panel-bg); border-bottom: 1px solid var(--border-color); display: flex; justify-content: space-between; align-items: center; height: 60px; box-sizing: border-box; z-index: 10; }\n    main { flex: 1; display: flex; overflow: hidden; position: relative; }\n    #graph-container { flex: 1; position: relative; overflow: hidden; background: var(--bg-color); }\n    #details-panel { width: var(--sidebar-width); background: var(--panel-bg); border-left: 1px solid var(--border-color); padding: 20px; overflow-y: auto; box-sizing: border-box; display: none; flex-direction: column; gap: 15px; }\n    #details-panel.visible { display: flex; }\n    footer { padding: 5px 20px; background: var(--panel-bg); border-top: 1px solid var(--border-color); font-size: 0.8rem; text-align: center; color: #666; height: 30px; display: flex; align-items: center; justify-content: center; }\n\n    /* Header Components */\n    .brand { font-weight: bold; font-size: 1.2rem; display: flex; align-items: center; gap: 10px; }\n    .brand span { color: var(--accent-color); }\n    #metrics-summary { font-size: 0.9rem; color: #aaa; display: flex; gap: 20px; }\n    .metric { display: flex; flex-direction: column; align-items: center; line-height: 1.1; }\n    .metric-value { font-weight: bold; color: var(--text-color); }\n    .metric-label { font-size: 0.7rem; }\n\n    #controls { display: flex; gap: 10px; align-items: center; }\n    .btn-group { display: flex; background: #333; border-radius: 4px; overflow: hidden; }\n    button { background: transparent; color: #aaa; border: none; padding: 6px 12px; cursor: pointer; font-size: 0.85rem; transition: all 0.2s; }\n    button:hover { color: white; background: rgba(255,255,255,0.1); }\n    button.active { background: var(--accent-color); color: white; }\n\n    /* Search */\n    #search-container { position: absolute; top: 15px; left: 15px; z-index: 5; }\n    #search-input { background: rgba(30,30,30,0.9); border: 1px solid #444; color: white; padding: 8px 12px; border-radius: 20px; width: 200px; outline: none; transition: width 0.3s; }\n    #search-input:focus { width: 280px; border-color: var(--accent-color); }\n\n    /* Graph */\n    svg { width: 100%; height: 100%; display: block; }\n    .node { cursor: pointer; transition: stroke-width 0.1s; }\n    .link { stroke: #555; stroke-opacity: 0.3; fill: none; pointer-events: none; }\n\n    /* Interaction States */\n    .node.highlight { stroke: #fff; stroke-width: 2px; }\n    .link.highlight { stroke-opacity: 0.8; stroke: #999; }\n    .node.faded { opacity: 0.1; }\n    .link.faded { opacity: 0.05; }\n\n    /* Details Panel Content */\n    .detail-section { border-bottom: 1px solid #333; padding-bottom: 10px; }\n    .detail-section:last-child { border-bottom: none; }\n    .detail-label { font-size: 0.75rem; color: #888; text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 4px; }\n    .detail-value { font-size: 0.95rem; word-break: break-all; }\n    .detail-list { list-style: none; padding: 0; margin: 0; max-height: 150px; overflow-y: auto; font-size: 0.85rem; }\n    .detail-list li { padding: 4px 0; border-bottom: 1px solid #2a2a2a; }\n    .detail-list a { color: var(--accent-color); text-decoration: none; }\n    .detail-list a:hover { text-decoration: underline; }\n\n    .status-badge { display: inline-block; padding: 2px 6px; border-radius: 3px; font-size: 0.75rem; font-weight: bold; margin-top: 5px; }\n    .status-ok { background: #2e7d32; color: white; }\n    .status-warn { background: #f9a825; color: black; }\n    .status-error { background: #c62828; color: white; }\n\n    /* Tooltip */\n    #tooltip { position: absolute; background: rgba(20,20,20,0.95); color: white; padding: 10px; border-radius: 6px; pointer-events: none; font-size: 12px; z-index: 100; box-shadow: 0 4px 15px rgba(0,0,0,0.5); border: 1px solid #444; display: none; transform: translate(-50%, -100%); margin-top: -10px; white-space: nowrap; }\n\n    /* Responsive Sidebar */\n    @media (max-width: 768px) {\n      #details-panel { position: absolute; right: 0; top: 0; bottom: 0; z-index: 20; box-shadow: -5px 0 15px rgba(0,0,0,0.5); transform: translateX(100%); transition: transform 0.3s ease; }\n      #details-panel.visible { transform: translateX(0); }\n      #metrics-summary { display: none; }\n    }\n  </style>\n</head>\n<body>\n  <header>\n    <div class=\"brand\"><span>Crawlith</span> SiteGraph</div>\n\n    <div id=\"metrics-summary\">\n      <div class=\"metric\"><span class=\"metric-value\" id=\"m-pages\">-</span><span class=\"metric-label\">Pages</span></div>\n      <div class=\"metric\"><span class=\"metric-value\" id=\"m-depth\">-</span><span class=\"metric-label\">Max Depth</span></div>\n      <div class=\"metric\"><span class=\"metric-value\" id=\"m-eff\">-</span><span class=\"metric-label\">Efficiency</span></div>\n      <div class=\"metric\"><span class=\"metric-value\" id=\"m-orphan\">-</span><span class=\"metric-label\">Orphans</span></div>\n    </div>\n\n    <div id=\"controls\">\n      <div class=\"btn-group\" style=\"margin-right: 15px;\">\n        <button id=\"btn-auth-pagerank\" class=\"active\" title=\"PageRank Authority\">PageRank</button>\n        <button id=\"btn-auth-structural\" title=\"Structural Authority (In-Degree)\">In-Degree</button>\n      </div>\n      <div class=\"btn-group\">\n        <button id=\"btn-hierarchical\" class=\"active\">Hierarchical</button>\n        <button id=\"btn-radial\">Radial</button>\n      </div>\n    </div>\n  </header>\n\n  <main>\n    <div id=\"graph-container\">\n      <div id=\"search-container\">\n        <input type=\"text\" id=\"search-input\" placeholder=\"Search URL...\">\n      </div>\n      <svg id=\"graph\"></svg>\n      <div id=\"tooltip\"></div>\n    </div>\n\n    <aside id=\"details-panel\">\n      <div class=\"detail-section\">\n        <div class=\"detail-label\">URL</div>\n        <div class=\"detail-value\" id=\"d-url\">-</div>\n        <div id=\"d-status\"></div>\n      </div>\n      <div class=\"detail-section\" style=\"display: flex; gap: 20px;\">\n        <div>\n          <div class=\"detail-label\">Depth</div>\n          <div class=\"detail-value\" id=\"d-depth\">-</div>\n        </div>\n        <div>\n          <div class=\"detail-label\">Authority</div>\n          <div class=\"detail-value\" id=\"d-auth-container\">-</div>\n        </div>\n      </div>\n      <div class=\"detail-section\">\n        <div class=\"detail-label\">In-links (<span id=\"d-in-count\">0</span>)</div>\n         <!-- List could be populated here if we had the reverse index, for now just count -->\n      </div>\n      <div class=\"detail-section\">\n        <div class=\"detail-label\">Out-links (<span id=\"d-out-count\">0</span>)</div>\n        <ul class=\"detail-list\" id=\"d-out-list\"></ul>\n      </div>\n    </aside>\n  </main>\n\n  <footer>\n    Generated by Crawlith Crawler\n  </footer>\n\n  <!-- D3 from CDN -->\n  <script src=\"https://d3js.org/d3.v7.min.js\"></script>\n\n  <script>\n    // --- State ---\n    const state = {\n      nodes: [],\n      links: [],\n      metrics: {},\n      adjacency: new Map(), // url -> { in: [], out: [] }\n      simulation: null,\n      width: 0,\n      height: 0,\n      transform: d3.zoomIdentity,\n      activeNode: null,\n      mode: 'hierarchical', // 'hierarchical' | 'radial'\n      maxDepth: 0,\n      maxInLinks: 0,\n      nodeSelection: null,\n      linkSelection: null,\n      zoom: null\n    };\n\n    // --- DOM Elements ---\n    const svg = d3.select(\"#graph\");\n    const container = svg.append(\"g\");\n    const linkGroup = container.append(\"g\").attr(\"class\", \"links\");\n    const nodeGroup = container.append(\"g\").attr(\"class\", \"nodes\");\n    const tooltip = d3.select(\"#tooltip\");\n    const detailsPanel = d3.select(\"#details-panel\");\n\n    // --- Initialization ---\n    // --- Initialization ---\n    async function init() {\n      try {\n        let graphData, metricsData;\n\n        // 1. Try to use injected data (for file:// usage)\n        // @ts-ignore\n        if (window.GRAPH_DATA) graphData = window.GRAPH_DATA;\n        // @ts-ignore\n        if (window.METRICS_DATA) metricsData = window.METRICS_DATA;\n\n        // 2. Fallback to fetching JSON files (for web server usage)\n        if (!graphData || !metricsData) {\n            try {\n                const [graphRes, metricsRes] = await Promise.all([\n                    fetch('graph.json'),\n                    fetch('metrics.json')\n                ]);\n                if (graphRes.ok && metricsRes.ok) {\n                    graphData = await graphRes.json();\n                    metricsData = await metricsRes.json();\n                }\n            } catch (e) {\n                console.warn(\"Fetch failed, possibly due to CORS or missing files.\", e);\n            }\n        }\n\n        if (!graphData || !metricsData) {\n            throw new Error(\"No data available. Ensure graph.json exists or data is injected.\");\n        }\n\n        state.metrics = metricsData;\n        processData(graphData);\n        updateMetricsUI();\n\n        // Setup UI\n        setupResize();\n        setupInteractions();\n        setupSearch();\n\n        // Start Simulation\n        initSimulation();\n\n      } catch (err) {\n        console.error(err);\n        alert(\"Error loading visualization data: \" + err.message);\n      }\n    }\n\n    function processData(data) {\n      // Create a map for fast lookup\n      const nodeMap = new Map();\n\n      data.nodes.forEach(n => {\n        n.inLinks = n.inLinks || 0;\n        n.outLinks = n.outLinks || 0;\n        nodeMap.set(n.url, n);\n      });\n\n      // Filter valid links\n      state.links = data.edges\n        .map(e => ({ source: nodeMap.get(e.source), target: nodeMap.get(e.target) }))\n        .filter(e => e.source && e.target);\n\n      state.nodes = data.nodes;\n\n      // Calculate Stats\n      state.maxDepth = d3.max(state.nodes, d => d.depth) || 1;\n      state.maxInLinks = d3.max(state.nodes, d => d.inLinks) || 1;\n\n      // Calculate Authority & Enrich Nodes\n      state.nodes.forEach(n => {\n        // Structural Authority: log-scaled normalized 0-1 based on in-links\n        n.structuralAuthority = Math.log(1 + n.inLinks) / Math.log(1 + state.maxInLinks);\n\n        // PageRank Authority: normalized 0-1 from pageRankScore (0-100)\n        if (typeof n.pageRankScore === 'number') {\n          n.pageRankAuthority = n.pageRankScore / 100;\n        } else {\n          n.pageRankAuthority = n.structuralAuthority;\n        }\n\n        // Default authority to PageRank if available, else structural\n        n.authority = n.pageRankAuthority;\n\n        // Ensure x,y are initialized to avoid NaNs if D3 doesn't do it fast enough\n        n.x = 0; n.y = 0;\n      });\n\n      // Build Adjacency Map\n      state.nodes.forEach(n => state.adjacency.set(n.url, { in: [], out: [] }));\n      state.links.forEach(l => {\n        state.adjacency.get(l.source.url).out.push(l.target);\n        state.adjacency.get(l.target.url).in.push(l.source);\n      });\n    }\n\n    function updateMetricsUI() {\n      document.getElementById('m-pages').textContent = state.metrics.totalPages;\n      document.getElementById('m-depth').textContent = state.metrics.maxDepthFound;\n      document.getElementById('m-eff').textContent = (state.metrics.crawlEfficiencyScore * 100).toFixed(1) + '%';\n      document.getElementById('m-orphan').textContent = state.metrics.orphanPages.length;\n    }\n\n    // --- Simulation ---\n    function initSimulation() {\n      const { width, height } = getDimensions();\n      state.width = width;\n      state.height = height;\n\n      // Safeguards\n      const nodeCount = state.nodes.length;\n      const enableCollision = nodeCount <= 1200;\n      const alphaDecay = nodeCount > 1000 ? 0.05 : 0.02; // Faster decay for large graphs\n\n      state.simulation = d3.forceSimulation(state.nodes)\n        .alphaDecay(alphaDecay)\n        .force(\"link\", d3.forceLink(state.links).id(d => d.url).strength(0.5)) // Reduced strength for flexibility\n        .force(\"charge\", d3.forceManyBody().strength(nodeCount > 1000 ? -100 : -300))\n        .force(\"center\", d3.forceCenter(width / 2, height / 2));\n\n      if (enableCollision) {\n        state.simulation.force(\"collide\", d3.forceCollide().radius(d => getNodeRadius(d) + 2).iterations(1));\n      }\n\n      // Apply Layout Mode\n      applyLayoutMode(state.mode);\n\n      // Rendering loop\n      state.simulation.on(\"tick\", ticked);\n\n      // Render initial SVG elements\n      render();\n    }\n\n    function applyLayoutMode(mode) {\n      state.mode = mode;\n      const { width, height } = state;\n      const centerY = height / 2;\n      const centerX = width / 2;\n\n      // Remove conflicting forces\n      state.simulation.force(\"y\", null);\n      state.simulation.force(\"radial\", null);\n\n      if (mode === 'hierarchical') {\n        const depthSpacing = height / (state.maxDepth + 2);\n        // Hierarchical: Nodes pushed to Y levels based on depth\n        state.simulation.force(\"y\", d3.forceY(d => {\n           return (d.depth * depthSpacing) - (height/2) + 50; // Offset to start from top\n        }).strength(1));\n        // We rely on \"center\" force to keep X centered, but maybe add weak forceX?\n        // Let's add weak forceX to prevent wide spread\n        state.simulation.force(\"x\", d3.forceX(0).strength(0.05));\n        state.simulation.force(\"center\", d3.forceCenter(width/2, height/2)); // Recenter\n\n      } else if (mode === 'radial') {\n        const maxRadius = Math.min(width, height) / 2 - 50;\n        const ringSpacing = maxRadius / (state.maxDepth + 1);\n\n        state.simulation.force(\"radial\", d3.forceRadial(\n          d => d.depth * ringSpacing,\n          width / 2,\n          height / 2\n        ).strength(0.8));\n\n        state.simulation.force(\"x\", null); // Remove X constraint\n      }\n\n      state.simulation.alpha(1).restart();\n    }\n\n    function getNodeRadius(d) {\n      // 5 + authority * 15\n      return 5 + (d.authority * 15);\n    }\n\n    function getNodeColor(d) {\n      // Depth-based sequential color (Blue -> Purple -> Pink)\n      const t = d.depth / (state.maxDepth || 1);\n      return d3.interpolateViridis(1 - t); // Invert Viridis for better contrast on dark\n    }\n\n    function render() {\n      // Links\n      state.linkSelection = linkGroup.selectAll(\"line\")\n        .data(state.links)\n        .join(\"line\")\n        .attr(\"class\", \"link\")\n        .attr(\"stroke-width\", 0.5);\n\n      // Nodes\n      state.nodeSelection = nodeGroup.selectAll(\"circle\")\n        .data(state.nodes)\n        .join(\"circle\")\n        .attr(\"class\", \"node\")\n        .attr(\"r\", d => getNodeRadius(d))\n        .attr(\"fill\", d => getNodeColor(d))\n        .attr(\"stroke\", d => d.status >= 400 ? \"#ff4444\" : null) // Red stroke for errors\n        .on(\"mouseover\", (event, d) => {\n          if (state.activeNode) return;\n          highlightNode(d);\n          showTooltip(event, d);\n        })\n        .on(\"mouseout\", () => {\n          if (state.activeNode) return;\n          resetHighlight();\n          hideTooltip();\n        })\n        .on(\"click\", (event, d) => {\n          event.stopPropagation();\n          selectNode(d);\n        })\n        .call(d3.drag()\n          .on(\"start\", dragstarted)\n          .on(\"drag\", dragged)\n          .on(\"end\", dragended));\n\n      // Zoom\n      state.zoom = d3.zoom()\n        .scaleExtent([0.1, 4])\n        .on(\"zoom\", (event) => {\n          state.transform = event.transform;\n          container.attr(\"transform\", event.transform);\n        });\n\n      svg.call(state.zoom)\n         .call(state.zoom.transform, d3.zoomIdentity.translate(state.width/2, state.height/2).scale(0.8).translate(-state.width/2, -state.height/2)); // Initial zoom out\n    }\n\n    function ticked() {\n      if (state.linkSelection) {\n        state.linkSelection\n          .attr(\"x1\", d => d.source.x)\n          .attr(\"y1\", d => d.source.y)\n          .attr(\"x2\", d => d.target.x)\n          .attr(\"y2\", d => d.target.y);\n      }\n\n      if (state.nodeSelection) {\n        state.nodeSelection\n          .attr(\"cx\", d => d.x)\n          .attr(\"cy\", d => d.y);\n      }\n    }\n\n    // --- Interactions ---\n\n    function setupInteractions() {\n      // Background click to clear selection\n      svg.on(\"click\", () => {\n        state.activeNode = null;\n        resetHighlight();\n        detailsPanel.classed(\"visible\", false);\n      });\n\n      // Layout Toggle\n      d3.select(\"#btn-hierarchical\").on(\"click\", function() {\n        setMode('hierarchical', this);\n      });\n      d3.select(\"#btn-radial\").on(\"click\", function() {\n        setMode('radial', this);\n      });\n\n      // Authority Toggle\n      d3.select(\"#btn-auth-pagerank\").on(\"click\", function() {\n        setAuthorityMode('pagerank', this);\n      });\n      d3.select(\"#btn-auth-structural\").on(\"click\", function() {\n        setAuthorityMode('structural', this);\n      });\n    }\n\n    function setAuthorityMode(mode, btn) {\n      d3.select(\"#btn-auth-pagerank\").classed(\"active\", false);\n      d3.select(\"#btn-auth-structural\").classed(\"active\", false);\n      d3.select(btn).classed(\"active\", true);\n\n      state.nodes.forEach(n => {\n        n.authority = mode === 'pagerank' ? n.pageRankAuthority : n.structuralAuthority;\n      });\n\n      // Update Visuals\n      nodeGroup.selectAll(\"circle\")\n        .transition().duration(500)\n        .attr(\"r\", d => getNodeRadius(d));\n\n      // Update collision force if enabled\n      if (state.simulation.force(\"collide\")) {\n        state.simulation.force(\"collide\", d3.forceCollide().radius(d => getNodeRadius(d) + 2).iterations(1));\n        state.simulation.alpha(0.3).restart();\n      }\n    }\n\n    function setMode(mode, btn) {\n      d3.selectAll(\"#controls button\").classed(\"active\", false);\n      d3.select(btn).classed(\"active\", true);\n      applyLayoutMode(mode);\n    }\n\n    function highlightNode(d) {\n      const neighbors = new Set();\n      const adj = state.adjacency.get(d.url);\n      if (adj) {\n        adj.in.forEach(n => neighbors.add(n.url));\n        adj.out.forEach(n => neighbors.add(n.url));\n      }\n      neighbors.add(d.url);\n\n      nodeGroup.selectAll(\"circle\").classed(\"faded\", n => !neighbors.has(n.url));\n      nodeGroup.selectAll(\"circle\").classed(\"highlight\", n => n.url === d.url);\n\n      linkGroup.selectAll(\"line\").classed(\"faded\", l =>\n        l.source.url !== d.url && l.target.url !== d.url\n      );\n      linkGroup.selectAll(\"line\").classed(\"highlight\", l =>\n        l.source.url === d.url || l.target.url === d.url\n      );\n    }\n\n    function resetHighlight() {\n      nodeGroup.selectAll(\"circle\").classed(\"faded\", false).classed(\"highlight\", false);\n      linkGroup.selectAll(\"line\").classed(\"faded\", false).classed(\"highlight\", false);\n    }\n\n    function selectNode(d) {\n      state.activeNode = d;\n      highlightNode(d);\n      showDetails(d);\n    }\n\n    function showTooltip(event, d) {\n      // If we are transforming the container, we need to map coordinates correctly or just use pageX/Y\n      tooltip.style(\"display\", \"block\")\n        .html(`<strong>${new URL(d.url).pathname}</strong><br>Auth: ${(d.authority * 10).toFixed(1)}`)\n        .style(\"left\", (event.pageX) + \"px\")\n        .style(\"top\", (event.pageY - 10) + \"px\");\n    }\n\n    function hideTooltip() {\n      tooltip.style(\"display\", \"none\");\n    }\n\n    function showDetails(d) {\n      detailsPanel.classed(\"visible\", true);\n      d3.select(\"#d-url\").text(d.url);\n      d3.select(\"#d-depth\").text(d.depth);\n\n      const authContainer = d3.select(\"#d-auth-container\");\n      authContainer.html(\"\");\n      const prVal = (d.pageRankAuthority * 100).toFixed(1);\n      const structVal = d.structuralAuthority.toFixed(3);\n      authContainer.append(\"div\").html(`PR: <strong>${prVal}</strong>`);\n      authContainer.append(\"div\").style(\"color\", \"#888\").style(\"font-size\", \"0.8em\").text(`In-Degree: ${structVal}`);\n\n      d3.select(\"#d-in-count\").text(d.inLinks);\n      d3.select(\"#d-out-count\").text(d.outLinks);\n\n      // Status badge\n      const statusDiv = d3.select(\"#d-status\");\n      statusDiv.html(\"\");\n      let sClass = \"status-ok\";\n      if (d.status >= 400) sClass = \"status-error\";\n      else if (d.status >= 300) sClass = \"status-warn\";\n      statusDiv.append(\"span\").attr(\"class\", \"status-badge \" + sClass).text(d.status);\n\n      // Outlinks list (limit to 20)\n      const list = d3.select(\"#d-out-list\");\n      list.html(\"\");\n      const adj = state.adjacency.get(d.url);\n      if (adj && adj.out.length > 0) {\n        adj.out.slice(0, 50).forEach(target => {\n          list.append(\"li\").append(\"a\")\n            .attr(\"href\", target.url)\n            .attr(\"target\", \"_blank\")\n            .text(new URL(target.url).pathname);\n        });\n        if (adj.out.length > 50) {\n            list.append(\"li\").text(`...and ${adj.out.length - 50} more`);\n        }\n      } else {\n        list.append(\"li\").text(\"No outgoing links\");\n      }\n    }\n\n    // --- Search ---\n    function setupSearch() {\n      const input = document.getElementById('search-input');\n      input.addEventListener('keydown', (e) => {\n        if (e.key === 'Enter') {\n          const val = input.value.trim().toLowerCase();\n          if (!val) return;\n\n          const found = state.nodes.find(n => n.url.toLowerCase().includes(val));\n          if (found) {\n            selectNode(found);\n            // Center view on node\n            const transform = d3.zoomIdentity\n              .translate(state.width/2, state.height/2)\n              .scale(2)\n              .translate(-found.x, -found.y);\n\n            svg.transition().duration(750).call(state.zoom.transform, transform);\n          }\n        }\n      });\n    }\n\n    function setupResize() {\n      window.addEventListener(\"resize\", () => {\n        const { width, height } = getDimensions();\n        state.width = width;\n        state.height = height;\n        state.simulation.force(\"center\", d3.forceCenter(width / 2, height / 2));\n        if (state.mode === 'hierarchical') {\n            // Re-evaluate Y force if needed, but usually center is enough\n        }\n        state.simulation.alpha(0.3).restart();\n      });\n    }\n\n    function getDimensions() {\n      const rect = document.getElementById(\"graph-container\").getBoundingClientRect();\n      return { width: rect.width, height: rect.height };\n    }\n\n    // --- Dragging ---\n    function dragstarted(event, d) {\n      if (!event.active) state.simulation.alphaTarget(0.3).restart();\n      d.fx = d.x;\n      d.fy = d.y;\n    }\n\n    function dragged(event, d) {\n      d.fx = event.x;\n      d.fy = event.y;\n    }\n\n    function dragended(event, d) {\n      if (!event.active) state.simulation.alphaTarget(0);\n      d.fx = null;\n      d.fy = null;\n    }\n\n    // Start\n    if (document.readyState === 'loading') {\n        document.addEventListener('DOMContentLoaded', init);\n    } else {\n        init();\n    }\n  </script>\n</body>\n</html>\n";