npm - @crawlith/core - Versions diffs - 0.1.0 → 0.1.2 - Mend

@crawlith/core 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (238) hide show

package/LICENSE +201 -0
package/README.md +70 -0
package/dist/analysis/analysis_list.html +35 -0
package/dist/analysis/analysis_page.html +123 -0
package/dist/analysis/analyze.d.ts +40 -5
package/dist/analysis/analyze.js +395 -347
package/dist/analysis/clustering.d.ts +23 -0
package/dist/analysis/clustering.js +206 -0
package/dist/analysis/content.d.ts +1 -1
package/dist/analysis/content.js +11 -5
package/dist/analysis/duplicate.d.ts +34 -0
package/dist/analysis/duplicate.js +305 -0
package/dist/analysis/heading.d.ts +116 -0
package/dist/analysis/heading.js +356 -0
package/dist/analysis/images.d.ts +1 -1
package/dist/analysis/images.js +6 -5
package/dist/analysis/links.d.ts +1 -1
package/dist/analysis/links.js +8 -8
package/dist/{scoring/orphanSeverity.d.ts → analysis/orphan.d.ts} +12 -23
package/dist/{scoring/orphanSeverity.js → analysis/orphan.js} +9 -3
package/dist/analysis/scoring.js +11 -2
package/dist/analysis/seo.d.ts +8 -4
package/dist/analysis/seo.js +41 -30
package/dist/analysis/soft404.d.ts +17 -0
package/dist/analysis/soft404.js +62 -0
package/dist/analysis/structuredData.d.ts +1 -1
package/dist/analysis/structuredData.js +5 -4
package/dist/analysis/templates.d.ts +2 -0
package/dist/analysis/templates.js +7 -0
package/dist/application/index.d.ts +2 -0
package/dist/application/index.js +2 -0
package/dist/application/usecase.d.ts +3 -0
package/dist/application/usecase.js +1 -0
package/dist/application/usecases.d.ts +114 -0
package/dist/application/usecases.js +201 -0
package/dist/audit/index.js +1 -1
package/dist/audit/transport.d.ts +1 -1
package/dist/audit/transport.js +5 -4
package/dist/audit/types.d.ts +1 -0
package/dist/constants.d.ts +17 -0
package/dist/constants.js +23 -0
package/dist/core/scope/scopeManager.js +3 -0
package/dist/core/security/ipGuard.d.ts +11 -0
package/dist/core/security/ipGuard.js +71 -3
package/dist/crawler/crawl.d.ts +4 -22
package/dist/crawler/crawl.js +4 -335
package/dist/crawler/crawler.d.ts +87 -0
package/dist/crawler/crawler.js +683 -0
package/dist/crawler/extract.d.ts +4 -1
package/dist/crawler/extract.js +7 -2
package/dist/crawler/fetcher.d.ts +2 -1
package/dist/crawler/fetcher.js +26 -11
package/dist/crawler/metricsRunner.d.ts +23 -1
package/dist/crawler/metricsRunner.js +202 -72
package/dist/crawler/normalize.d.ts +41 -0
package/dist/crawler/normalize.js +119 -3
package/dist/crawler/parser.d.ts +1 -3
package/dist/crawler/parser.js +2 -49
package/dist/crawler/resolver.d.ts +11 -0
package/dist/crawler/resolver.js +67 -0
package/dist/crawler/sitemap.d.ts +6 -0
package/dist/crawler/sitemap.js +27 -17
package/dist/crawler/trap.d.ts +5 -1
package/dist/crawler/trap.js +23 -2
package/dist/db/CrawlithDB.d.ts +110 -0
package/dist/db/CrawlithDB.js +500 -0
package/dist/db/graphLoader.js +42 -30
package/dist/db/index.d.ts +11 -0
package/dist/db/index.js +41 -29
package/dist/db/migrations.d.ts +2 -0
package/dist/db/{schema.js → migrations.js} +90 -43
package/dist/db/pluginRegistry.d.ts +9 -0
package/dist/db/pluginRegistry.js +19 -0
package/dist/db/repositories/EdgeRepository.d.ts +13 -0
package/dist/db/repositories/EdgeRepository.js +20 -0
package/dist/db/repositories/MetricsRepository.d.ts +16 -8
package/dist/db/repositories/MetricsRepository.js +28 -7
package/dist/db/repositories/PageRepository.d.ts +15 -2
package/dist/db/repositories/PageRepository.js +169 -25
package/dist/db/repositories/SiteRepository.d.ts +9 -0
package/dist/db/repositories/SiteRepository.js +13 -0
package/dist/db/repositories/SnapshotRepository.d.ts +14 -5
package/dist/db/repositories/SnapshotRepository.js +64 -5
package/dist/db/reset.d.ts +9 -0
package/dist/db/reset.js +32 -0
package/dist/db/statements.d.ts +12 -0
package/dist/db/statements.js +40 -0
package/dist/diff/compare.d.ts +0 -5
package/dist/diff/compare.js +0 -12
package/dist/diff/service.d.ts +16 -0
package/dist/diff/service.js +41 -0
package/dist/domain/index.d.ts +4 -0
package/dist/domain/index.js +4 -0
package/dist/events.d.ts +56 -0
package/dist/events.js +1 -0
package/dist/graph/graph.d.ts +36 -42
package/dist/graph/graph.js +26 -17
package/dist/graph/hits.d.ts +23 -0
package/dist/graph/hits.js +111 -0
package/dist/graph/metrics.d.ts +0 -4
package/dist/graph/metrics.js +25 -9
package/dist/graph/pagerank.d.ts +17 -4
package/dist/graph/pagerank.js +126 -91
package/dist/graph/simhash.d.ts +6 -0
package/dist/graph/simhash.js +14 -0
package/dist/index.d.ts +29 -8
package/dist/index.js +29 -8
package/dist/lock/hashKey.js +1 -1
package/dist/lock/lockManager.d.ts +5 -1
package/dist/lock/lockManager.js +38 -13
package/dist/plugin-system/plugin-cli.d.ts +10 -0
package/dist/plugin-system/plugin-cli.js +31 -0
package/dist/plugin-system/plugin-config.d.ts +16 -0
package/dist/plugin-system/plugin-config.js +36 -0
package/dist/plugin-system/plugin-loader.d.ts +17 -0
package/dist/plugin-system/plugin-loader.js +122 -0
package/dist/plugin-system/plugin-registry.d.ts +25 -0
package/dist/plugin-system/plugin-registry.js +167 -0
package/dist/plugin-system/plugin-types.d.ts +205 -0
package/dist/plugin-system/plugin-types.js +1 -0
package/dist/ports/index.d.ts +9 -0
package/dist/ports/index.js +1 -0
package/{src/report/sitegraph_template.ts → dist/report/crawl.html} +330 -81
package/dist/report/crawlExport.d.ts +3 -0
package/dist/report/{sitegraphExport.js → crawlExport.js} +3 -3
package/dist/report/crawl_template.d.ts +1 -0
package/dist/report/crawl_template.js +7 -0
package/dist/report/export.d.ts +3 -0
package/dist/report/export.js +81 -0
package/dist/report/html.js +15 -216
package/dist/report/insight.d.ts +27 -0
package/dist/report/insight.js +103 -0
package/dist/scoring/health.d.ts +56 -0
package/dist/scoring/health.js +213 -0
package/dist/utils/chalk.d.ts +6 -0
package/dist/utils/chalk.js +41 -0
package/dist/utils/secureConfig.d.ts +23 -0
package/dist/utils/secureConfig.js +128 -0
package/package.json +12 -6
package/CHANGELOG.md +0 -7
package/dist/db/schema.d.ts +0 -2
package/dist/graph/cluster.d.ts +0 -6
package/dist/graph/cluster.js +0 -173
package/dist/graph/duplicate.d.ts +0 -10
package/dist/graph/duplicate.js +0 -251
package/dist/report/sitegraphExport.d.ts +0 -3
package/dist/report/sitegraph_template.d.ts +0 -1
package/dist/report/sitegraph_template.js +0 -630
package/dist/scoring/hits.d.ts +0 -9
package/dist/scoring/hits.js +0 -111
package/src/analysis/analyze.ts +0 -548
package/src/analysis/content.ts +0 -62
package/src/analysis/images.ts +0 -28
package/src/analysis/links.ts +0 -41
package/src/analysis/scoring.ts +0 -59
package/src/analysis/seo.ts +0 -82
package/src/analysis/structuredData.ts +0 -62
package/src/audit/dns.ts +0 -49
package/src/audit/headers.ts +0 -98
package/src/audit/index.ts +0 -66
package/src/audit/scoring.ts +0 -232
package/src/audit/transport.ts +0 -258
package/src/audit/types.ts +0 -102
package/src/core/network/proxyAdapter.ts +0 -21
package/src/core/network/rateLimiter.ts +0 -39
package/src/core/network/redirectController.ts +0 -47
package/src/core/network/responseLimiter.ts +0 -34
package/src/core/network/retryPolicy.ts +0 -57
package/src/core/scope/domainFilter.ts +0 -45
package/src/core/scope/scopeManager.ts +0 -52
package/src/core/scope/subdomainPolicy.ts +0 -39
package/src/core/security/ipGuard.ts +0 -92
package/src/crawler/crawl.ts +0 -382
package/src/crawler/extract.ts +0 -34
package/src/crawler/fetcher.ts +0 -233
package/src/crawler/metricsRunner.ts +0 -124
package/src/crawler/normalize.ts +0 -108
package/src/crawler/parser.ts +0 -190
package/src/crawler/sitemap.ts +0 -73
package/src/crawler/trap.ts +0 -96
package/src/db/graphLoader.ts +0 -105
package/src/db/index.ts +0 -70
package/src/db/repositories/EdgeRepository.ts +0 -29
package/src/db/repositories/MetricsRepository.ts +0 -49
package/src/db/repositories/PageRepository.ts +0 -128
package/src/db/repositories/SiteRepository.ts +0 -32
package/src/db/repositories/SnapshotRepository.ts +0 -74
package/src/db/schema.ts +0 -177
package/src/diff/compare.ts +0 -84
package/src/graph/cluster.ts +0 -192
package/src/graph/duplicate.ts +0 -286
package/src/graph/graph.ts +0 -172
package/src/graph/metrics.ts +0 -110
package/src/graph/pagerank.ts +0 -125
package/src/graph/simhash.ts +0 -61
package/src/index.ts +0 -30
package/src/lock/hashKey.ts +0 -51
package/src/lock/lockManager.ts +0 -124
package/src/lock/pidCheck.ts +0 -13
package/src/report/html.ts +0 -227
package/src/report/sitegraphExport.ts +0 -58
package/src/scoring/hits.ts +0 -131
package/src/scoring/orphanSeverity.ts +0 -176
package/src/utils/version.ts +0 -18
package/tests/__snapshots__/orphanSeverity.test.ts.snap +0 -49
package/tests/analysis.unit.test.ts +0 -98
package/tests/analyze.integration.test.ts +0 -98
package/tests/audit/dns.test.ts +0 -31
package/tests/audit/headers.test.ts +0 -45
package/tests/audit/scoring.test.ts +0 -133
package/tests/audit/security.test.ts +0 -12
package/tests/audit/transport.test.ts +0 -112
package/tests/clustering.test.ts +0 -118
package/tests/crawler.test.ts +0 -358
package/tests/db.test.ts +0 -159
package/tests/diff.test.ts +0 -67
package/tests/duplicate.test.ts +0 -110
package/tests/fetcher.test.ts +0 -106
package/tests/fetcher_safety.test.ts +0 -85
package/tests/fixtures/analyze-crawl.json +0 -26
package/tests/hits.test.ts +0 -134
package/tests/html_report.test.ts +0 -58
package/tests/lock/lockManager.test.ts +0 -138
package/tests/metrics.test.ts +0 -196
package/tests/normalize.test.ts +0 -101
package/tests/orphanSeverity.test.ts +0 -160
package/tests/pagerank.test.ts +0 -98
package/tests/parser.test.ts +0 -117
package/tests/proxy_safety.test.ts +0 -57
package/tests/redirect_safety.test.ts +0 -73
package/tests/safety.test.ts +0 -114
package/tests/scope.test.ts +0 -66
package/tests/scoring.test.ts +0 -59
package/tests/sitemap.test.ts +0 -88
package/tests/soft404.test.ts +0 -41
package/tests/trap.test.ts +0 -39
package/tests/visualization_data.test.ts +0 -46
package/tsconfig.json +0 -11

package/dist/report/html.js CHANGED Viewed

@@ -1,223 +1,22 @@
+import { Crawl_HTML } from './crawl_template.js';
 function safeJson(data) {
     return JSON.stringify(data).replace(/</g, '\\u003c');
 }
 export function generateHtml(graphData, metrics) {
-    const graphJson = safeJson(graphData);
-    return `<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Crawlith Site Graph</title>
-    <style>
-        body { margin: 0; overflow: hidden; font-family: sans-serif; }
-        #graph { width: 100vw; height: 100vh; background: #f0f0f0; }
-        .tooltip {
-            position: absolute;
-            background: white;
-            border: 1px solid #ccc;
-            padding: 10px;
-            pointer-events: none;
-            font-size: 12px;
-            box-shadow: 2px 2px 5px rgba(0,0,0,0.1);
-            display: none;
-        }
-        #metrics {
-            position: absolute;
-            top: 10px;
-            left: 10px;
-            background: rgba(255, 255, 255, 0.9);
-            padding: 15px;
-            border-radius: 5px;
-            box-shadow: 0 0 10px rgba(0,0,0,0.1);
-            max-width: 320px;
-            max-height: 90vh;
-            overflow-y: auto;
-            z-index: 100;
-        }
-        h1 { font-size: 18px; margin-top: 0; }
-        h2 { font-size: 14px; margin: 15px 0 5px; border-bottom: 1px solid #ddd; }
-        ul { padding-left: 20px; margin: 5px 0; }
-        .legend { margin-top: 10px; font-size: 11px; }
-        .legend-item { display: flex; align-items: center; margin-bottom: 3px; }
-        .dot { width: 8px; height: 8px; border-radius: 50%; margin-right: 5px; }
-        .stat-row { display: flex; justify-content: space-between; font-size: 13px; margin-bottom: 3px; }
-        .stat-label { color: #666; }
-        .stat-value { font-weight: bold; }
-    </style>
-</head>
-<body>
-    <div id="metrics">
-        <h1>Crawlith Site Graph</h1>
-        <div class="stat-row">
-            <span class="stat-label">Discovered Pages:</span>
-            <span class="stat-value">${metrics.totalPages}</span>
-        </div>
-        ${metrics.sessionStats ? `
-        <div class="stat-row">
-            <span class="stat-label">Session Crawl:</span>
-            <span class="stat-value">${metrics.sessionStats.pagesFetched} pages</span>
-        </div>
-        ${metrics.sessionStats.pagesCached > 0 ? `
-        <div class="stat-row" style="font-size: 11px; margin-top: -3px;">
-            <span class="stat-label" style="padding-left: 10px;">- Reuse Cached:</span>
-            <span class="stat-value">${metrics.sessionStats.pagesCached}</span>
-        </div>` : ''}
-        ` : ''}
-        <div class="stat-row">
-            <span class="stat-label">Total Edges:</span>
-            <span class="stat-value">${metrics.totalEdges}</span>
-        </div>
-        <div class="stat-row">
-            <span class="stat-label">Max Depth:</span>
-            <span class="stat-value">${metrics.maxDepthFound}</span>
-        </div>
-        <div class="stat-row">
-            <span class="stat-label">Avg Out-Degree:</span>
-            <span class="stat-value">${metrics.averageOutDegree.toFixed(2)}</span>
-        </div>
-        <div class="legend">
-            <div class="legend-item"><div class="dot" style="background: red;"></div>Orphan (In-Links: 0)</div>
-            <div class="legend-item"><div class="dot" style="background: orange;"></div>Deep (Depth >= 4)</div>
-            <div class="legend-item"><div class="dot" style="background: blue;"></div>Normal</div>
-        </div>
-        ${metrics.topAuthorityPages.length > 0 ? `
-        <h3>Top Authority</h3>
-        <ul>
-            ${metrics.topAuthorityPages.map(p => `<li><a href="${p.url}" target="_blank">${new URL(p.url).pathname}</a> (${p.authority.toFixed(2)})</li>`).join('')}
-        </ul>
-        ` : ''}
-        ${metrics.orphanPages.length > 0 ? `
-        <h3>Orphan Pages (${metrics.orphanPages.length})</h3>
-        <details>
-            <summary>Show list</summary>
-            <ul>
-                ${metrics.orphanPages.slice(0, 20).map(url => `<li><a href="${url}" target="_blank">${url}</a></li>`).join('')}
-                ${metrics.orphanPages.length > 20 ? `<li>... and ${metrics.orphanPages.length - 20} more</li>` : ''}
-            </ul>
-        </details>
-        ` : ''}
-    </div>
-    <div id="graph"></div>
-    <div class="tooltip" id="tooltip"></div>
-    <script src="https://d3js.org/d3.v7.min.js"></script>
-    <script>
-        // Make data available globally
+    // Strip heavy HTML content from nodes to keep the report lightweight
+    const vizGraphData = {
+        ...graphData,
+        nodes: graphData.nodes ? graphData.nodes.map((n) => {
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const { html, ...rest } = n;
+            return rest;
+        }) : []
+    };
+    const graphJson = safeJson(vizGraphData);
+    const metricsJson = safeJson(metrics);
+    return Crawl_HTML.replace('</body>', `<script>
         window.GRAPH_DATA = ${graphJson};
-        const data = window.GRAPH_DATA;
-        const width = window.innerWidth;
-        const height = window.innerHeight;
-        const svg = d3.select("#graph").append("svg")
-            .attr("width", width)
-            .attr("height", height)
-            .call(d3.zoom().on("zoom", (event) => {
-                g.attr("transform", event.transform);
-            }));
-        const g = svg.append("g");
-        // Define arrow marker
-        svg.append("defs").selectAll("marker")
-            .data(["arrow"])
-            .enter().append("marker")
-            .attr("id", d => d)
-            .attr("viewBox", "0 -5 10 10")
-            .attr("refX", 15)
-            .attr("refY", 0)
-            .attr("markerWidth", 6)
-            .attr("markerHeight", 6)
-            .attr("orient", "auto")
-            .append("path")
-            .attr("d", "M0,-5L10,0L0,5")
-            .attr("fill", "#999");
-        const simulation = d3.forceSimulation(data.nodes)
-            .force("link", d3.forceLink(data.edges).id(d => d.url).distance(100))
-            .force("charge", d3.forceManyBody().strength(-300))
-            .force("center", d3.forceCenter(width / 2, height / 2))
-            .force("collide", d3.forceCollide().radius(d => Math.sqrt((d.inLinks || 0) + 1) * 5 + 2));
-        const link = g.append("g")
-            .attr("stroke", "#999")
-            .attr("stroke-opacity", 0.6)
-            .selectAll("line")
-            .data(data.edges)
-            .join("line")
-            .attr("stroke-width", 1)
-            .attr("marker-end", "url(#arrow)");
-        const node = g.append("g")
-            .attr("stroke", "#fff")
-            .attr("stroke-width", 1.5)
-            .selectAll("circle")
-            .data(data.nodes)
-            .join("circle")
-            .attr("r", d => Math.sqrt((d.inLinks || 0) + 1) * 3 + 2)
-            .attr("fill", d => {
-                if (d.inLinks === 0 && d.depth > 0) return "red";
-                if (d.depth >= 4) return "orange";
-                return "blue";
-            })
-            .call(d3.drag()
-                .on("start", dragstarted)
-                .on("drag", dragged)
-                .on("end", dragended));
-        const tooltip = d3.select("#tooltip");
-        node.on("mouseover", (event, d) => {
-            tooltip.style("display", "block")
-                .html(\`
-                    <strong>URL:</strong> \${d.url}<br>
-                    <strong>Depth:</strong> \${d.depth}<br>
-                    <strong>In-Links:</strong> \${d.inLinks}<br>
-                    <strong>Out-Links:</strong> \${d.outLinks}<br>
-                    <strong>Status:</strong> \${d.status}
-                \`)
-                .style("left", (event.pageX + 10) + "px")
-                .style("top", (event.pageY - 10) + "px");
-        })
-        .on("mouseout", () => {
-            tooltip.style("display", "none");
-        });
-        simulation.on("tick", () => {
-            link
-                .attr("x1", d => d.source.x)
-                .attr("y1", d => d.source.y)
-                .attr("x2", d => d.target.x)
-                .attr("y2", d => d.target.y);
-            node
-                .attr("cx", d => d.x)
-                .attr("cy", d => d.y);
-        });
-        function dragstarted(event, d) {
-            if (!event.active) simulation.alphaTarget(0.3).restart();
-            d.fx = d.x;
-            d.fy = d.y;
-        }
-        function dragged(event, d) {
-            d.fx = event.x;
-            d.fy = event.y;
-        }
-        function dragended(event, d) {
-            if (!event.active) simulation.alphaTarget(0);
-            d.fx = null;
-            d.fy = null;
-        }
+        window.METRICS_DATA = ${metricsJson};
     </script>
-</body>
-</html>`;
+    </body>`);
 }

package/dist/report/insight.d.ts ADDED Viewed

@@ -0,0 +1,27 @@
+import { Graph, Metrics } from '@crawlith/core';
+export interface CrawlInsightReport {
+    pages: number;
+    fetchedPages?: number;
+    summary: {
+        crawlDepth: number;
+        internalLinks: number;
+        externalLinks: number;
+    };
+    health?: {
+        score: number;
+        status: string;
+        weightedPenalties: any;
+    };
+    issues?: any;
+    topAuthorityPages: {
+        url: string;
+        score: number;
+    }[];
+}
+export declare function buildCrawlInsightReport(graph: Graph, metrics: Metrics, healthData?: {
+    health: any;
+    issues: any;
+}): CrawlInsightReport;
+export declare function renderInsightOutput(report: CrawlInsightReport, snapshotId: number): string;
+export declare function renderScoreBreakdown(health: any): string;
+export declare function hasCriticalIssues(report: CrawlInsightReport): boolean;

package/dist/report/insight.js ADDED Viewed

@@ -0,0 +1,103 @@
+export function buildCrawlInsightReport(graph, metrics, healthData) {
+    return {
+        pages: metrics.totalPages,
+        fetchedPages: metrics.sessionStats?.pagesFetched,
+        health: healthData?.health,
+        issues: healthData?.issues,
+        summary: {
+            crawlDepth: metrics.maxDepthFound,
+            internalLinks: metrics.totalEdges,
+            externalLinks: healthData?.issues?.externalLinks || 0
+        },
+        topAuthorityPages: metrics.topAuthorityPages.map(p => ({ url: p.url, score: p.authority }))
+    };
+}
+export function renderInsightOutput(report, snapshotId) {
+    const lines = [];
+    // Header
+    lines.push(`CRAWLITH — Crawl`);
+    lines.push('');
+    lines.push(`# ${snapshotId}`);
+    lines.push('');
+    if (report.fetchedPages !== undefined) {
+        if (report.fetchedPages === report.pages) {
+            lines.push(`${report.pages} pages crawled`);
+        }
+        else {
+            lines.push(`${report.fetchedPages} pages fetched / ${report.pages} discovered`);
+        }
+    }
+    else {
+        lines.push(`${report.pages} pages crawled`);
+    }
+    lines.push('');
+    // Health Score if available
+    if (report.health) {
+        lines.push(`Score: ${report.health.score} (${report.health.status})`);
+        lines.push('');
+    }
+    // ===== Critical =====
+    if (report.issues) {
+        const critical = [];
+        const addLine = (arr, condition, text) => condition && arr.push(text);
+        addLine(critical, report.issues.orphanPages > 0, `${report.issues.orphanPages} orphan pages`);
+        addLine(critical, report.issues.redirectChains > 0, `${report.issues.redirectChains} redirect chains`);
+        addLine(critical, report.issues.brokenInternalLinks > 0, `${report.issues.brokenInternalLinks} broken internal links`);
+        addLine(critical, report.issues.duplicateClusters > 0, `${report.issues.duplicateClusters} near-duplicate clusters`);
+        addLine(critical, report.issues.canonicalConflicts > 0, `${report.issues.canonicalConflicts} canonical conflicts`);
+        addLine(critical, report.issues.accidentalNoindex > 0, `${report.issues.accidentalNoindex} pages accidentally noindexed`);
+        addLine(critical, report.issues.blockedByRobots > 0, `${report.issues.blockedByRobots} pages blocked by robots.txt`);
+        if (critical.length > 0) {
+            lines.push(`Critical`);
+            for (const c of critical)
+                lines.push(`  • ${c}`);
+            lines.push('');
+        }
+        // ===== Warnings =====
+        const warnings = [];
+        addLine(warnings, report.issues.missingH1 > 0, `${report.issues.missingH1} pages missing H1`);
+        addLine(warnings, report.issues.thinContent > 0, `${report.issues.thinContent} thin content pages`);
+        addLine(warnings, report.issues.excessiveInternalLinkCount > 0, `${report.issues.excessiveInternalLinkCount} pages with excessive links`);
+        addLine(warnings, report.issues.imageAltMissing > 0, `${report.issues.imageAltMissing} pages missing image alt`);
+        if (warnings.length > 0) {
+            lines.push(`Warnings`);
+            for (const w of warnings)
+                lines.push(`  • ${w}`);
+            lines.push('');
+        }
+    }
+    // ===== Structure =====
+    lines.push(`Structure`);
+    lines.push(`  Depth Reached     ${report.summary.crawlDepth}`);
+    lines.push(`  Internal Links    ${report.summary.internalLinks}`);
+    lines.push(`  External Links    ${report.summary.externalLinks}`);
+    lines.push('');
+    // ===== Authority =====
+    if (report.topAuthorityPages.length > 0) {
+        lines.push(`Top Authority`);
+        for (const page of report.topAuthorityPages.slice(0, 10)) {
+            lines.push(`  ${page.url}   ${page.score.toFixed(3)}`);
+        }
+        lines.push('');
+    }
+    return `${lines.join('\n')}\n`;
+}
+export function renderScoreBreakdown(health) {
+    return [
+        'Health Score Breakdown',
+        `weights: ${JSON.stringify(health.weights)}`,
+        `penalties: ${JSON.stringify(health.weightedPenalties)}`
+    ].join('\n');
+}
+export function hasCriticalIssues(report) {
+    if (!report.issues)
+        return false;
+    const { issues } = report;
+    return (issues.orphanPages > 0 ||
+        issues.brokenInternalLinks > 0 ||
+        issues.redirectChains > 0 ||
+        issues.duplicateClusters > 0 ||
+        issues.canonicalConflicts > 0 ||
+        issues.accidentalNoindex > 0 ||
+        issues.blockedByRobots > 0);
+}

package/dist/scoring/health.d.ts ADDED Viewed

@@ -0,0 +1,56 @@
+import { Graph } from '../graph/graph.js';
+export interface HealthScoreWeights {
+    orphans: number;
+    brokenLinks: number;
+    redirectChains: number;
+    duplicateClusters: number;
+    thinContent: number;
+    missingH1: number;
+    noindexMisuse: number;
+    canonicalConflicts: number;
+    lowInternalLinks: number;
+    excessiveLinks: number;
+    blockedByRobots: number;
+    crawlTraps: number;
+}
+export interface CrawlIssueCounts {
+    orphanPages: number;
+    brokenInternalLinks: number;
+    redirectChains: number;
+    duplicateClusters: number;
+    canonicalConflicts: number;
+    accidentalNoindex: number;
+    missingH1: number;
+    thinContent: number;
+    lowInternalLinkCount: number;
+    excessiveInternalLinkCount: number;
+    highExternalLinkRatio: number;
+    imageAltMissing: number;
+    strongPagesUnderLinking: number;
+    cannibalizationClusters: number;
+    nearAuthorityThreshold: number;
+    underlinkedHighAuthorityPages: number;
+    externalLinks: number;
+    blockedByRobots: number;
+    crawlTraps: number;
+}
+export interface HealthScoreBreakdown {
+    score: number;
+    status: string;
+    weightedPenalties: Record<string, number>;
+    weights: HealthScoreWeights;
+}
+export declare const THIN_CONTENT_THRESHOLD = 200;
+export declare const LOW_INTERNAL_LINK_THRESHOLD = 2;
+export declare const EXCESSIVE_INTERNAL_LINK_THRESHOLD = 150;
+export declare const HIGH_EXTERNAL_LINK_RATIO_THRESHOLD = 0.6;
+export declare const OPPORTUNITY_AUTHORITY_THRESHOLD = 0.8;
+export declare const DEFAULT_HEALTH_WEIGHTS: HealthScoreWeights;
+export declare class HealthService {
+    calculateHealthScore(totalPages: number, issues: Pick<CrawlIssueCounts, 'orphanPages' | 'brokenInternalLinks' | 'redirectChains' | 'duplicateClusters' | 'thinContent' | 'missingH1' | 'accidentalNoindex' | 'canonicalConflicts' | 'lowInternalLinkCount' | 'excessiveInternalLinkCount' | 'blockedByRobots' | 'crawlTraps'>, weights?: HealthScoreWeights): HealthScoreBreakdown;
+    collectCrawlIssues(graph: Graph, metrics: any, rootOrigin?: string): CrawlIssueCounts;
+    private clamp;
+    private healthStatusLabel;
+}
+export declare const calculateHealthScore: (totalPages: number, issues: Pick<CrawlIssueCounts, "orphanPages" | "brokenInternalLinks" | "redirectChains" | "duplicateClusters" | "thinContent" | "missingH1" | "accidentalNoindex" | "canonicalConflicts" | "lowInternalLinkCount" | "excessiveInternalLinkCount" | "blockedByRobots" | "crawlTraps">, weights?: HealthScoreWeights) => HealthScoreBreakdown;
+export declare const healthStatusLabel: (score: number, hasCritical?: boolean) => "Needs Attention" | "Excellent" | "Good" | "Critical";

package/dist/scoring/health.js ADDED Viewed

@@ -0,0 +1,213 @@
+import { analyzeContent } from '../analysis/content.js';
+import { analyzeH1 } from '../analysis/seo.js';
+import { analyzeImageAlts } from '../analysis/images.js';
+import { analyzeLinks } from '../analysis/links.js';
+import { UrlUtil } from '../crawler/normalize.js';
+export const THIN_CONTENT_THRESHOLD = 200;
+export const LOW_INTERNAL_LINK_THRESHOLD = 2;
+export const EXCESSIVE_INTERNAL_LINK_THRESHOLD = 150;
+export const HIGH_EXTERNAL_LINK_RATIO_THRESHOLD = 0.6;
+export const OPPORTUNITY_AUTHORITY_THRESHOLD = 0.8;
+export const DEFAULT_HEALTH_WEIGHTS = {
+    orphans: 50,
+    brokenLinks: 100,
+    redirectChains: 20,
+    duplicateClusters: 25,
+    thinContent: 15,
+    missingH1: 10,
+    noindexMisuse: 20,
+    canonicalConflicts: 10,
+    lowInternalLinks: 10,
+    excessiveLinks: 5,
+    blockedByRobots: 100,
+    crawlTraps: 50
+};
+export class HealthService {
+    calculateHealthScore(totalPages, issues, weights = DEFAULT_HEALTH_WEIGHTS) {
+        const safePages = Math.max(totalPages, 1);
+        const weightedPenalties = {
+            orphans: this.clamp(((issues.orphanPages || 0) / safePages) * weights.orphans, 0, weights.orphans),
+            brokenLinks: this.clamp(((issues.brokenInternalLinks || 0) / safePages) * weights.brokenLinks, 0, weights.brokenLinks),
+            redirectChains: this.clamp(((issues.redirectChains || 0) / safePages) * weights.redirectChains, 0, weights.redirectChains),
+            duplicateClusters: this.clamp(((issues.duplicateClusters || 0) / safePages) * weights.duplicateClusters, 0, weights.duplicateClusters),
+            thinContent: this.clamp(((issues.thinContent || 0) / safePages) * weights.thinContent, 0, weights.thinContent),
+            missingH1: this.clamp(((issues.missingH1 || 0) / safePages) * weights.missingH1, 0, weights.missingH1),
+            noindexMisuse: this.clamp(((issues.accidentalNoindex || 0) / safePages) * weights.noindexMisuse, 0, weights.noindexMisuse),
+            canonicalConflicts: this.clamp(((issues.canonicalConflicts || 0) / safePages) * weights.canonicalConflicts, 0, weights.canonicalConflicts),
+            lowInternalLinks: this.clamp(((issues.lowInternalLinkCount || 0) / safePages) * weights.lowInternalLinks, 0, weights.lowInternalLinks),
+            excessiveLinks: this.clamp(((issues.excessiveInternalLinkCount || 0) / safePages) * weights.excessiveLinks, 0, weights.excessiveLinks),
+            blockedByRobots: this.clamp(((issues.blockedByRobots || 0) / safePages) * weights.blockedByRobots, 0, weights.blockedByRobots),
+            crawlTraps: this.clamp(((issues.crawlTraps || 0) / safePages) * weights.crawlTraps, 0, weights.crawlTraps)
+        };
+        const totalPenalty = Object.values(weightedPenalties).reduce((sum, value) => sum + value, 0);
+        const score = Number(this.clamp(100 - totalPenalty, 0, 100).toFixed(1));
+        const hasCritical = ((issues.orphanPages || 0) > 0 ||
+            (issues.brokenInternalLinks || 0) > 0 ||
+            (issues.redirectChains || 0) > 0 ||
+            (issues.duplicateClusters || 0) > 0 ||
+            (issues.canonicalConflicts || 0) > 0 ||
+            (issues.accidentalNoindex || 0) > 0 ||
+            (issues.blockedByRobots || 0) > 0);
+        return {
+            score,
+            status: this.healthStatusLabel(score, hasCritical),
+            weightedPenalties,
+            weights
+        };
+    }
+    collectCrawlIssues(graph, metrics, rootOrigin = '') {
+        const nodes = graph.getNodes();
+        let brokenInternalLinks = 0;
+        let redirectChains = 0;
+        let canonicalConflicts = 0;
+        let accidentalNoindex = 0;
+        let missingH1 = 0;
+        let thinContent = 0;
+        let highExternalLinkRatio = 0;
+        let imageAltMissing = 0;
+        let lowInternalLinkCount = 0;
+        let excessiveInternalLinkCount = 0;
+        let strongPagesUnderLinking = 0;
+        let nearAuthorityThreshold = 0;
+        let underlinkedHighAuthorityPages = 0;
+        let externalLinks = 0;
+        let blockedByRobots = 0;
+        let crawlTraps = 0;
+        for (const node of nodes) {
+            if (!node.isInternal) {
+                continue;
+            }
+            if (node.crawlStatus === 'blocked' || node.crawlStatus === 'blocked_by_robots') {
+                blockedByRobots += 1;
+            }
+            if (node.crawlTrapFlag) {
+                crawlTraps += 1;
+            }
+            const isConfirmedError = node.status >= 400 || (node.status === 0 && (node.crawlStatus === 'network_error' || node.crawlStatus === 'failed_after_retries' || node.securityError || node.crawlStatus === 'fetched_error'));
+            if (isConfirmedError) {
+                brokenInternalLinks += 1;
+            }
+            if (node.brokenLinks) {
+                const actualBreaks = node.brokenLinks.filter(url => {
+                    const target = graph.nodes.get(url);
+                    return target && (target.status >= 400 || (target.status === 0 && (target.crawlStatus === 'network_error' || target.crawlStatus === 'failed_after_retries' || target.securityError || target.crawlStatus === 'fetched_error')));
+                });
+                brokenInternalLinks += actualBreaks.length;
+            }
+            if ((node.redirectChain?.length || 0) > 1) {
+                redirectChains += 1;
+            }
+            const absoluteUrl = rootOrigin ? (node.url.startsWith('http') ? node.url : new URL(node.url, rootOrigin).toString()) : node.url;
+            if (node.canonical && node.canonical !== node.url && node.canonical !== absoluteUrl) {
+                // Final check: normalize both to ignore trailing slash differences or protocol mismatches if they are considered "same"
+                const normCanonical = node.canonical.replace(/\/$/, '');
+                const normAbsolute = absoluteUrl.replace(/\/$/, '');
+                if (normCanonical !== normAbsolute) {
+                    canonicalConflicts += 1;
+                }
+            }
+            if (node.noindex && node.status >= 200 && node.status < 300) {
+                accidentalNoindex += 1;
+            }
+            if (node.inLinks === 1 && node.depth > 0) {
+                lowInternalLinkCount += 1;
+            }
+            if (node.outLinks > EXCESSIVE_INTERNAL_LINK_THRESHOLD) {
+                excessiveInternalLinkCount += 1;
+            }
+            if (!node.html) {
+                continue;
+            }
+            const h1Res = analyzeH1(node.html, '');
+            if (h1Res.count === 0) {
+                missingH1 += 1;
+            }
+            if (node.wordCount != null) {
+                if (node.wordCount < THIN_CONTENT_THRESHOLD) {
+                    thinContent += 1;
+                }
+            }
+            else if (node.html) {
+                const content = analyzeContent(node.html);
+                if (content.wordCount < THIN_CONTENT_THRESHOLD) {
+                    thinContent += 1;
+                }
+            }
+            const pageAbsUrl = rootOrigin ? UrlUtil.toAbsolute(node.url, rootOrigin) : node.url;
+            const links = analyzeLinks(node.html || '', pageAbsUrl, rootOrigin || node.url);
+            externalLinks += links.externalLinks;
+            if (links.externalRatio > HIGH_EXTERNAL_LINK_RATIO_THRESHOLD) {
+                highExternalLinkRatio += 1;
+            }
+            if (node.html) {
+                const imageAlt = analyzeImageAlts(node.html);
+                if (imageAlt.missingAlt > 0) {
+                    imageAltMissing += 1;
+                }
+            }
+        }
+        const clusters = graph.contentClusters || metrics.clusters || [];
+        const duplicateClusters = clusters.length;
+        const cannibalizationClusters = clusters.filter((cluster) => cluster.risk === 'high' || cluster.type === 'near').length;
+        for (const node of nodes) {
+            const authority = node.inLinks > 5 ? 0.8 : 0.2;
+            if (authority >= OPPORTUNITY_AUTHORITY_THRESHOLD && node.outLinks < 3) {
+                strongPagesUnderLinking += 1;
+            }
+            if (authority >= 0.65 && authority < OPPORTUNITY_AUTHORITY_THRESHOLD) {
+                nearAuthorityThreshold += 1;
+            }
+            if (authority >= OPPORTUNITY_AUTHORITY_THRESHOLD && node.inLinks < LOW_INTERNAL_LINK_THRESHOLD) {
+                underlinkedHighAuthorityPages += 1;
+            }
+        }
+        return {
+            orphanPages: metrics.orphanPages?.length || 0,
+            brokenInternalLinks,
+            redirectChains,
+            duplicateClusters,
+            canonicalConflicts,
+            accidentalNoindex,
+            missingH1,
+            thinContent,
+            lowInternalLinkCount,
+            excessiveInternalLinkCount,
+            highExternalLinkRatio,
+            imageAltMissing,
+            strongPagesUnderLinking,
+            cannibalizationClusters,
+            nearAuthorityThreshold,
+            underlinkedHighAuthorityPages,
+            externalLinks,
+            blockedByRobots,
+            crawlTraps
+        };
+    }
+    clamp(value, min, max) {
+        return Math.min(max, Math.max(min, value));
+    }
+    healthStatusLabel(score, hasCritical = false) {
+        if (hasCritical && score >= 75)
+            return 'Needs Attention';
+        if (score >= 90)
+            return 'Excellent';
+        if (score >= 75)
+            return 'Good';
+        if (score >= 50)
+            return 'Needs Attention';
+        return 'Critical';
+    }
+}
+const service = new HealthService();
+export const calculateHealthScore = (totalPages, issues, weights = DEFAULT_HEALTH_WEIGHTS) => service.calculateHealthScore(totalPages, issues, weights);
+export const healthStatusLabel = (score, hasCritical = false) => {
+    if (hasCritical && score >= 75)
+        return 'Needs Attention';
+    if (score >= 90)
+        return 'Excellent';
+    if (score >= 75)
+        return 'Good';
+    if (score >= 50)
+        return 'Needs Attention';
+    return 'Critical';
+};

package/dist/utils/chalk.d.ts ADDED Viewed

@@ -0,0 +1,6 @@
+interface Chalk {
+    (text: unknown): string;
+    [key: string]: Chalk;
+}
+declare const chalk: Chalk;
+export default chalk;