npm - @crawlith/core - Versions diffs - 0.1.0 → 0.1.1 - Mend

@crawlith/core 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (131) hide show

package/CHANGELOG.md +6 -0
package/dist/analysis/analysis_list.html +35 -0
package/dist/analysis/analysis_page.html +123 -0
package/dist/analysis/analyze.d.ts +17 -3
package/dist/analysis/analyze.js +192 -248
package/dist/analysis/scoring.js +7 -1
package/dist/analysis/templates.d.ts +2 -0
package/dist/analysis/templates.js +7 -0
package/dist/core/security/ipGuard.d.ts +11 -0
package/dist/core/security/ipGuard.js +71 -3
package/dist/crawler/crawl.d.ts +4 -22
package/dist/crawler/crawl.js +4 -335
package/dist/crawler/crawler.d.ts +75 -0
package/dist/crawler/crawler.js +518 -0
package/dist/crawler/extract.d.ts +4 -1
package/dist/crawler/extract.js +7 -2
package/dist/crawler/fetcher.d.ts +1 -0
package/dist/crawler/fetcher.js +20 -5
package/dist/crawler/metricsRunner.d.ts +3 -1
package/dist/crawler/metricsRunner.js +55 -46
package/dist/crawler/sitemap.d.ts +3 -0
package/dist/crawler/sitemap.js +5 -1
package/dist/db/graphLoader.js +32 -3
package/dist/db/index.d.ts +3 -0
package/dist/db/index.js +4 -0
package/dist/db/repositories/EdgeRepository.d.ts +8 -0
package/dist/db/repositories/EdgeRepository.js +13 -0
package/dist/db/repositories/MetricsRepository.d.ts +3 -0
package/dist/db/repositories/MetricsRepository.js +14 -1
package/dist/db/repositories/PageRepository.d.ts +11 -0
package/dist/db/repositories/PageRepository.js +112 -19
package/dist/db/repositories/SiteRepository.d.ts +3 -0
package/dist/db/repositories/SiteRepository.js +9 -0
package/dist/db/repositories/SnapshotRepository.d.ts +2 -0
package/dist/db/repositories/SnapshotRepository.js +23 -2
package/dist/events.d.ts +48 -0
package/dist/events.js +1 -0
package/dist/graph/cluster.js +62 -14
package/dist/graph/duplicate.js +242 -191
package/dist/graph/graph.d.ts +16 -0
package/dist/graph/graph.js +17 -4
package/dist/graph/metrics.js +12 -0
package/dist/graph/pagerank.js +2 -0
package/dist/graph/simhash.d.ts +6 -0
package/dist/graph/simhash.js +14 -0
package/dist/index.d.ts +5 -2
package/dist/index.js +5 -2
package/dist/lock/hashKey.js +1 -1
package/dist/lock/lockManager.d.ts +4 -1
package/dist/lock/lockManager.js +23 -13
package/{src/report/sitegraph_template.ts → dist/report/crawl.html} +330 -81
package/dist/report/crawlExport.d.ts +3 -0
package/dist/report/{sitegraphExport.js → crawlExport.js} +3 -3
package/dist/report/crawl_template.d.ts +1 -0
package/dist/report/crawl_template.js +7 -0
package/dist/report/html.js +15 -216
package/dist/scoring/health.d.ts +50 -0
package/dist/scoring/health.js +170 -0
package/dist/scoring/hits.d.ts +1 -0
package/dist/scoring/hits.js +64 -44
package/dist/scoring/orphanSeverity.d.ts +5 -5
package/package.json +3 -3
package/scripts/copy-assets.js +37 -0
package/src/analysis/analysis_list.html +35 -0
package/src/analysis/analysis_page.html +123 -0
package/src/analysis/analyze.ts +218 -261
package/src/analysis/scoring.ts +8 -1
package/src/analysis/templates.ts +9 -0
package/src/core/security/ipGuard.ts +82 -3
package/src/crawler/crawl.ts +6 -379
package/src/crawler/crawler.ts +601 -0
package/src/crawler/extract.ts +7 -2
package/src/crawler/fetcher.ts +24 -6
package/src/crawler/metricsRunner.ts +60 -47
package/src/crawler/sitemap.ts +4 -1
package/src/db/graphLoader.ts +33 -3
package/src/db/index.ts +5 -0
package/src/db/repositories/EdgeRepository.ts +14 -0
package/src/db/repositories/MetricsRepository.ts +15 -1
package/src/db/repositories/PageRepository.ts +119 -19
package/src/db/repositories/SiteRepository.ts +11 -0
package/src/db/repositories/SnapshotRepository.ts +28 -3
package/src/events.ts +16 -0
package/src/graph/cluster.ts +69 -15
package/src/graph/duplicate.ts +249 -185
package/src/graph/graph.ts +24 -4
package/src/graph/metrics.ts +15 -0
package/src/graph/pagerank.ts +1 -0
package/src/graph/simhash.ts +15 -0
package/src/index.ts +5 -2
package/src/lock/hashKey.ts +1 -1
package/src/lock/lockManager.ts +21 -13
package/{dist/report/sitegraph_template.js → src/report/crawl.html} +330 -81
package/src/report/{sitegraphExport.ts → crawlExport.ts} +3 -3
package/src/report/crawl_template.ts +9 -0
package/src/report/html.ts +17 -217
package/src/scoring/health.ts +241 -0
package/src/scoring/hits.ts +67 -45
package/src/scoring/orphanSeverity.ts +8 -8
package/tests/analysis.unit.test.ts +44 -0
package/tests/analyze.integration.test.ts +88 -53
package/tests/analyze_markdown.test.ts +98 -0
package/tests/audit/audit.test.ts +101 -0
package/tests/audit/scoring.test.ts +25 -25
package/tests/audit/transport.test.ts +0 -1
package/tests/clustering_risk.test.ts +118 -0
package/tests/crawler.test.ts +19 -13
package/tests/db/index.test.ts +134 -0
package/tests/db/repositories.test.ts +115 -0
package/tests/db_repos.test.ts +72 -0
package/tests/duplicate.test.ts +2 -2
package/tests/extract.test.ts +86 -0
package/tests/fetcher.test.ts +5 -1
package/tests/fetcher_safety.test.ts +9 -3
package/tests/graph/graph.test.ts +100 -0
package/tests/graphLoader.test.ts +124 -0
package/tests/html_report.test.ts +52 -51
package/tests/ipGuard.test.ts +73 -0
package/tests/lock/lockManager.test.ts +77 -17
package/tests/normalize.test.ts +6 -19
package/tests/orphanSeverity.test.ts +9 -9
package/tests/redirect_safety.test.ts +5 -1
package/tests/renderAnalysisCsv.test.ts +183 -0
package/tests/safety.test.ts +12 -0
package/tests/scope.test.ts +18 -0
package/tests/scoring.test.ts +25 -24
package/tests/sitemap.test.ts +13 -1
package/tests/ssrf_fix.test.ts +69 -0
package/tests/visualization_data.test.ts +10 -10
package/dist/report/sitegraphExport.d.ts +0 -3
package/dist/report/sitegraph_template.d.ts +0 -1

package/{dist/report/sitegraph_template.js → src/report/crawl.html} RENAMED Viewed

@@ -1,5 +1,6 @@
-export const SITEGRAPH_HTML = `<!DOCTYPE html>
+<!DOCTYPE html>
 <html lang="en">
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
@@ -13,81 +14,329 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
       --accent-color: #4a90e2;
       --sidebar-width: 300px;
     }
-    body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; background: var(--bg-color); color: var(--text-color); height: 100vh; display: flex; flex-direction: column; overflow: hidden; }
+    body {
+      margin: 0;
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+      background: var(--bg-color);
+      color: var(--text-color);
+      height: 100vh;
+      display: flex;
+      flex-direction: column;
+      overflow: hidden;
+    }
     /* Layout */
-    header { padding: 0 20px; background: var(--panel-bg); border-bottom: 1px solid var(--border-color); display: flex; justify-content: space-between; align-items: center; height: 60px; box-sizing: border-box; z-index: 10; }
-    main { flex: 1; display: flex; overflow: hidden; position: relative; }
-    #graph-container { flex: 1; position: relative; overflow: hidden; background: var(--bg-color); }
-    #details-panel { width: var(--sidebar-width); background: var(--panel-bg); border-left: 1px solid var(--border-color); padding: 20px; overflow-y: auto; box-sizing: border-box; display: none; flex-direction: column; gap: 15px; }
-    #details-panel.visible { display: flex; }
-    footer { padding: 5px 20px; background: var(--panel-bg); border-top: 1px solid var(--border-color); font-size: 0.8rem; text-align: center; color: #666; height: 30px; display: flex; align-items: center; justify-content: center; }
+    header {
+      padding: 0 20px;
+      background: var(--panel-bg);
+      border-bottom: 1px solid var(--border-color);
+      display: flex;
+      justify-content: space-between;
+      align-items: center;
+      height: 60px;
+      box-sizing: border-box;
+      z-index: 10;
+    }
+    main {
+      flex: 1;
+      display: flex;
+      overflow: hidden;
+      position: relative;
+    }
+    #graph-container {
+      flex: 1;
+      position: relative;
+      overflow: hidden;
+      background: var(--bg-color);
+    }
+    #details-panel {
+      width: var(--sidebar-width);
+      background: var(--panel-bg);
+      border-left: 1px solid var(--border-color);
+      padding: 20px;
+      overflow-y: auto;
+      box-sizing: border-box;
+      display: none;
+      flex-direction: column;
+      gap: 15px;
+    }
+    #details-panel.visible {
+      display: flex;
+    }
+    footer {
+      padding: 5px 20px;
+      background: var(--panel-bg);
+      border-top: 1px solid var(--border-color);
+      font-size: 0.8rem;
+      text-align: center;
+      color: #666;
+      height: 30px;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+    }
     /* Header Components */
-    .brand { font-weight: bold; font-size: 1.2rem; display: flex; align-items: center; gap: 10px; }
-    .brand span { color: var(--accent-color); }
-    #metrics-summary { font-size: 0.9rem; color: #aaa; display: flex; gap: 20px; }
-    .metric { display: flex; flex-direction: column; align-items: center; line-height: 1.1; }
-    .metric-value { font-weight: bold; color: var(--text-color); }
-    .metric-label { font-size: 0.7rem; }
-    #controls { display: flex; gap: 10px; align-items: center; }
-    .btn-group { display: flex; background: #333; border-radius: 4px; overflow: hidden; }
-    button { background: transparent; color: #aaa; border: none; padding: 6px 12px; cursor: pointer; font-size: 0.85rem; transition: all 0.2s; }
-    button:hover { color: white; background: rgba(255,255,255,0.1); }
-    button.active { background: var(--accent-color); color: white; }
+    .brand {
+      font-weight: bold;
+      font-size: 1.2rem;
+      display: flex;
+      align-items: center;
+      gap: 10px;
+    }
+    .brand span {
+      color: var(--accent-color);
+    }
+    #metrics-summary {
+      font-size: 0.9rem;
+      color: #aaa;
+      display: flex;
+      gap: 20px;
+    }
+    .metric {
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      line-height: 1.1;
+    }
+    .metric-value {
+      font-weight: bold;
+      color: var(--text-color);
+    }
+    .metric-label {
+      font-size: 0.7rem;
+    }
+    #controls {
+      display: flex;
+      gap: 10px;
+      align-items: center;
+    }
+    .btn-group {
+      display: flex;
+      background: #333;
+      border-radius: 4px;
+      overflow: hidden;
+    }
+    button {
+      background: transparent;
+      color: #aaa;
+      border: none;
+      padding: 6px 12px;
+      cursor: pointer;
+      font-size: 0.85rem;
+      transition: all 0.2s;
+    }
+    button:hover {
+      color: white;
+      background: rgba(255, 255, 255, 0.1);
+    }
+    button.active {
+      background: var(--accent-color);
+      color: white;
+    }
     /* Search */
-    #search-container { position: absolute; top: 15px; left: 15px; z-index: 5; }
-    #search-input { background: rgba(30,30,30,0.9); border: 1px solid #444; color: white; padding: 8px 12px; border-radius: 20px; width: 200px; outline: none; transition: width 0.3s; }
-    #search-input:focus { width: 280px; border-color: var(--accent-color); }
+    #search-container {
+      position: absolute;
+      top: 15px;
+      left: 15px;
+      z-index: 5;
+    }
+    #search-input {
+      background: rgba(30, 30, 30, 0.9);
+      border: 1px solid #444;
+      color: white;
+      padding: 8px 12px;
+      border-radius: 20px;
+      width: 200px;
+      outline: none;
+      transition: width 0.3s;
+    }
+    #search-input:focus {
+      width: 280px;
+      border-color: var(--accent-color);
+    }
     /* Graph */
-    svg { width: 100%; height: 100%; display: block; }
-    .node { cursor: pointer; transition: stroke-width 0.1s; }
-    .link { stroke: #555; stroke-opacity: 0.3; fill: none; pointer-events: none; }
+    svg {
+      width: 100%;
+      height: 100%;
+      display: block;
+    }
+    .node {
+      cursor: pointer;
+      transition: stroke-width 0.1s;
+    }
+    .link {
+      stroke: #555;
+      stroke-opacity: 0.3;
+      fill: none;
+      pointer-events: none;
+    }
     /* Interaction States */
-    .node.highlight { stroke: #fff; stroke-width: 2px; }
-    .link.highlight { stroke-opacity: 0.8; stroke: #999; }
-    .node.faded { opacity: 0.1; }
-    .link.faded { opacity: 0.05; }
+    .node.highlight {
+      stroke: #fff;
+      stroke-width: 2px;
+    }
+    .link.highlight {
+      stroke-opacity: 0.8;
+      stroke: #999;
+    }
+    .node.faded {
+      opacity: 0.1;
+    }
+    .link.faded {
+      opacity: 0.05;
+    }
     /* Details Panel Content */
-    .detail-section { border-bottom: 1px solid #333; padding-bottom: 10px; }
-    .detail-section:last-child { border-bottom: none; }
-    .detail-label { font-size: 0.75rem; color: #888; text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 4px; }
-    .detail-value { font-size: 0.95rem; word-break: break-all; }
-    .detail-list { list-style: none; padding: 0; margin: 0; max-height: 150px; overflow-y: auto; font-size: 0.85rem; }
-    .detail-list li { padding: 4px 0; border-bottom: 1px solid #2a2a2a; }
-    .detail-list a { color: var(--accent-color); text-decoration: none; }
-    .detail-list a:hover { text-decoration: underline; }
-    .status-badge { display: inline-block; padding: 2px 6px; border-radius: 3px; font-size: 0.75rem; font-weight: bold; margin-top: 5px; }
-    .status-ok { background: #2e7d32; color: white; }
-    .status-warn { background: #f9a825; color: black; }
-    .status-error { background: #c62828; color: white; }
+    .detail-section {
+      border-bottom: 1px solid #333;
+      padding-bottom: 10px;
+    }
+    .detail-section:last-child {
+      border-bottom: none;
+    }
+    .detail-label {
+      font-size: 0.75rem;
+      color: #888;
+      text-transform: uppercase;
+      letter-spacing: 0.5px;
+      margin-bottom: 4px;
+    }
+    .detail-value {
+      font-size: 0.95rem;
+      word-break: break-all;
+    }
+    .detail-list {
+      list-style: none;
+      padding: 0;
+      margin: 0;
+      max-height: 150px;
+      overflow-y: auto;
+      font-size: 0.85rem;
+    }
+    .detail-list li {
+      padding: 4px 0;
+      border-bottom: 1px solid #2a2a2a;
+    }
+    .detail-list a {
+      color: var(--accent-color);
+      text-decoration: none;
+    }
+    .detail-list a:hover {
+      text-decoration: underline;
+    }
+    .status-badge {
+      display: inline-block;
+      padding: 2px 6px;
+      border-radius: 3px;
+      font-size: 0.75rem;
+      font-weight: bold;
+      margin-top: 5px;
+    }
+    .status-ok {
+      background: #2e7d32;
+      color: white;
+    }
+    .status-warn {
+      background: #f9a825;
+      color: black;
+    }
+    .status-error {
+      background: #c62828;
+      color: white;
+    }
     /* Tooltip */
-    #tooltip { position: absolute; background: rgba(20,20,20,0.95); color: white; padding: 10px; border-radius: 6px; pointer-events: none; font-size: 12px; z-index: 100; box-shadow: 0 4px 15px rgba(0,0,0,0.5); border: 1px solid #444; display: none; transform: translate(-50%, -100%); margin-top: -10px; white-space: nowrap; }
+    #tooltip {
+      position: absolute;
+      background: rgba(20, 20, 20, 0.95);
+      color: white;
+      padding: 10px;
+      border-radius: 6px;
+      pointer-events: none;
+      font-size: 12px;
+      z-index: 100;
+      box-shadow: 0 4px 15px rgba(0, 0, 0, 0.5);
+      border: 1px solid #444;
+      display: none;
+      transform: translate(-50%, -100%);
+      margin-top: -10px;
+      white-space: nowrap;
+    }
     /* Responsive Sidebar */
     @media (max-width: 768px) {
-      #details-panel { position: absolute; right: 0; top: 0; bottom: 0; z-index: 20; box-shadow: -5px 0 15px rgba(0,0,0,0.5); transform: translateX(100%); transition: transform 0.3s ease; }
-      #details-panel.visible { transform: translateX(0); }
-      #metrics-summary { display: none; }
+      #details-panel {
+        position: absolute;
+        right: 0;
+        top: 0;
+        bottom: 0;
+        z-index: 20;
+        box-shadow: -5px 0 15px rgba(0, 0, 0, 0.5);
+        transform: translateX(100%);
+        transition: transform 0.3s ease;
+      }
+      #details-panel.visible {
+        transform: translateX(0);
+      }
+      #metrics-summary {
+        display: none;
+      }
     }
   </style>
 </head>
 <body>
   <header>
-    <div class="brand"><span>Crawlith</span> SiteGraph</div>
+    <div class="brand"><span>Crawlith</span> Crawl</div>
     <div id="metrics-summary">
       <div class="metric"><span class="metric-value" id="m-pages">-</span><span class="metric-label">Pages</span></div>
-      <div class="metric"><span class="metric-value" id="m-depth">-</span><span class="metric-label">Max Depth</span></div>
-      <div class="metric"><span class="metric-value" id="m-eff">-</span><span class="metric-label">Efficiency</span></div>
-      <div class="metric"><span class="metric-value" id="m-orphan">-</span><span class="metric-label">Orphans</span></div>
+      <div class="metric"><span class="metric-value" id="m-depth">-</span><span class="metric-label">Max Depth</span>
+      </div>
+      <div class="metric"><span class="metric-value" id="m-eff">-</span><span class="metric-label">Efficiency</span>
+      </div>
+      <div class="metric"><span class="metric-value" id="m-orphan">-</span><span class="metric-label">Orphans</span>
+      </div>
     </div>
     <div id="controls">
@@ -129,7 +378,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
       </div>
       <div class="detail-section">
         <div class="detail-label">In-links (<span id="d-in-count">0</span>)</div>
-         <!-- List could be populated here if we had the reverse index, for now just count -->
+        <!-- List could be populated here if we had the reverse index, for now just count -->
       </div>
       <div class="detail-section">
         <div class="detail-label">Out-links (<span id="d-out-count">0</span>)</div>
@@ -187,22 +436,22 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
         // 2. Fallback to fetching JSON files (for web server usage)
         if (!graphData || !metricsData) {
-            try {
-                const [graphRes, metricsRes] = await Promise.all([
-                    fetch('graph.json'),
-                    fetch('metrics.json')
-                ]);
-                if (graphRes.ok && metricsRes.ok) {
-                    graphData = await graphRes.json();
-                    metricsData = await metricsRes.json();
-                }
-            } catch (e) {
-                console.warn("Fetch failed, possibly due to CORS or missing files.", e);
+          try {
+            const [graphRes, metricsRes] = await Promise.all([
+              fetch('graph.json'),
+              fetch('metrics.json')
+            ]);
+            if (graphRes.ok && metricsRes.ok) {
+              graphData = await graphRes.json();
+              metricsData = await metricsRes.json();
             }
+          } catch (e) {
+            console.warn("Fetch failed, possibly due to CORS or missing files.", e);
+          }
         }
         if (!graphData || !metricsData) {
-            throw new Error("No data available. Ensure graph.json exists or data is injected.");
+          throw new Error("No data available. Ensure graph.json exists or data is injected.");
         }
         state.metrics = metricsData;
@@ -323,12 +572,12 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
         const depthSpacing = height / (state.maxDepth + 2);
         // Hierarchical: Nodes pushed to Y levels based on depth
         state.simulation.force("y", d3.forceY(d => {
-           return (d.depth * depthSpacing) - (height/2) + 50; // Offset to start from top
+          return (d.depth * depthSpacing) - (height / 2) + 50; // Offset to start from top
         }).strength(1));
         // We rely on "center" force to keep X centered, but maybe add weak forceX?
         // Let's add weak forceX to prevent wide spread
         state.simulation.force("x", d3.forceX(0).strength(0.05));
-        state.simulation.force("center", d3.forceCenter(width/2, height/2)); // Recenter
+        state.simulation.force("center", d3.forceCenter(width / 2, height / 2)); // Recenter
       } else if (mode === 'radial') {
         const maxRadius = Math.min(width, height) / 2 - 50;
@@ -401,7 +650,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
         });
       svg.call(state.zoom)
-         .call(state.zoom.transform, d3.zoomIdentity.translate(state.width/2, state.height/2).scale(0.8).translate(-state.width/2, -state.height/2)); // Initial zoom out
+        .call(state.zoom.transform, d3.zoomIdentity.translate(state.width / 2, state.height / 2).scale(0.8).translate(-state.width / 2, -state.height / 2)); // Initial zoom out
     }
     function ticked() {
@@ -431,18 +680,18 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
       });
       // Layout Toggle
-      d3.select("#btn-hierarchical").on("click", function() {
+      d3.select("#btn-hierarchical").on("click", function () {
         setMode('hierarchical', this);
       });
-      d3.select("#btn-radial").on("click", function() {
+      d3.select("#btn-radial").on("click", function () {
         setMode('radial', this);
       });
       // Authority Toggle
-      d3.select("#btn-auth-pagerank").on("click", function() {
+      d3.select("#btn-auth-pagerank").on("click", function () {
         setAuthorityMode('pagerank', this);
       });
-      d3.select("#btn-auth-structural").on("click", function() {
+      d3.select("#btn-auth-structural").on("click", function () {
         setAuthorityMode('structural', this);
       });
     }
@@ -508,7 +757,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
     function showTooltip(event, d) {
       // If we are transforming the container, we need to map coordinates correctly or just use pageX/Y
       tooltip.style("display", "block")
-        .html(\`<strong>\${new URL(d.url).pathname}</strong><br>Auth: \${(d.authority * 10).toFixed(1)}\`)
+        .html(`<strong>${new URL(d.url).pathname}</strong><br>Auth: ${(d.authority * 10).toFixed(1)}`)
         .style("left", (event.pageX) + "px")
         .style("top", (event.pageY - 10) + "px");
     }
@@ -526,8 +775,8 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
       authContainer.html("");
       const prVal = (d.pageRankAuthority * 100).toFixed(1);
       const structVal = d.structuralAuthority.toFixed(3);
-      authContainer.append("div").html(\`PR: <strong>\${prVal}</strong>\`);
-      authContainer.append("div").style("color", "#888").style("font-size", "0.8em").text(\`In-Degree: \${structVal}\`);
+      authContainer.append("div").html(`PR: <strong>${prVal}</strong>`);
+      authContainer.append("div").style("color", "#888").style("font-size", "0.8em").text(`In-Degree: ${structVal}`);
       d3.select("#d-in-count").text(d.inLinks);
       d3.select("#d-out-count").text(d.outLinks);
@@ -552,7 +801,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
             .text(new URL(target.url).pathname);
         });
         if (adj.out.length > 50) {
-            list.append("li").text(\`...and \${adj.out.length - 50} more\`);
+          list.append("li").text(`...and ${adj.out.length - 50} more`);
         }
       } else {
         list.append("li").text("No outgoing links");
@@ -572,7 +821,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
             selectNode(found);
             // Center view on node
             const transform = d3.zoomIdentity
-              .translate(state.width/2, state.height/2)
+              .translate(state.width / 2, state.height / 2)
               .scale(2)
               .translate(-found.x, -found.y);
@@ -589,7 +838,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
         state.height = height;
         state.simulation.force("center", d3.forceCenter(width / 2, height / 2));
         if (state.mode === 'hierarchical') {
-            // Re-evaluate Y force if needed, but usually center is enough
+          // Re-evaluate Y force if needed, but usually center is enough
         }
         state.simulation.alpha(0.3).restart();
       });
@@ -620,11 +869,11 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
     // Start
     if (document.readyState === 'loading') {
-        document.addEventListener('DOMContentLoaded', init);
+      document.addEventListener('DOMContentLoaded', init);
     } else {
-        init();
+      init();
     }
   </script>
 </body>
-</html>
-`;
+</html>

package/src/report/{sitegraphExport.ts → crawlExport.ts} RENAMED Viewed

@@ -1,4 +1,4 @@
-export function renderSitegraphCsvNodes(graphData: any): string {
+export function renderCrawlCsvNodes(graphData: any): string {
     const nodeHeaders = ['URL', 'Depth', 'Status', 'InboundLinks', 'OutboundLinks', 'PageRankScore'];
     const nodeRows = graphData.nodes.map((n: any) => {
         const outbound = graphData.edges.filter((e: any) => e.source === n.url).length;
@@ -9,13 +9,13 @@ export function renderSitegraphCsvNodes(graphData: any): string {
     return [nodeHeaders.join(','), ...nodeRows].join('\n');
 }
-export function renderSitegraphCsvEdges(graphData: any): string {
+export function renderCrawlCsvEdges(graphData: any): string {
     const edgeHeaders = ['Source', 'Target', 'Weight'];
     const edgeRows = graphData.edges.map((e: any) => [e.source, e.target, e.weight].join(','));
     return [edgeHeaders.join(','), ...edgeRows].join('\n');
 }
-export function renderSitegraphMarkdown(url: string, graphData: any, metrics: any, graph: any): string {
+export function renderCrawlMarkdown(url: string, graphData: any, metrics: any, graph: any): string {
     const md = [
         `# Crawlith Crawl Summary - ${url}`,
         '',

package/src/report/crawl_template.ts ADDED Viewed

@@ -0,0 +1,9 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const templatePath = path.join(__dirname, 'crawl.html');
+export const Crawl_HTML = fs.readFileSync(templatePath, 'utf-8');