npm - @crawlith/core - Versions diffs - 0.1.0 → 0.1.2 - Mend

@crawlith/core 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (238) hide show

package/LICENSE +201 -0
package/README.md +70 -0
package/dist/analysis/analysis_list.html +35 -0
package/dist/analysis/analysis_page.html +123 -0
package/dist/analysis/analyze.d.ts +40 -5
package/dist/analysis/analyze.js +395 -347
package/dist/analysis/clustering.d.ts +23 -0
package/dist/analysis/clustering.js +206 -0
package/dist/analysis/content.d.ts +1 -1
package/dist/analysis/content.js +11 -5
package/dist/analysis/duplicate.d.ts +34 -0
package/dist/analysis/duplicate.js +305 -0
package/dist/analysis/heading.d.ts +116 -0
package/dist/analysis/heading.js +356 -0
package/dist/analysis/images.d.ts +1 -1
package/dist/analysis/images.js +6 -5
package/dist/analysis/links.d.ts +1 -1
package/dist/analysis/links.js +8 -8
package/dist/{scoring/orphanSeverity.d.ts → analysis/orphan.d.ts} +12 -23
package/dist/{scoring/orphanSeverity.js → analysis/orphan.js} +9 -3
package/dist/analysis/scoring.js +11 -2
package/dist/analysis/seo.d.ts +8 -4
package/dist/analysis/seo.js +41 -30
package/dist/analysis/soft404.d.ts +17 -0
package/dist/analysis/soft404.js +62 -0
package/dist/analysis/structuredData.d.ts +1 -1
package/dist/analysis/structuredData.js +5 -4
package/dist/analysis/templates.d.ts +2 -0
package/dist/analysis/templates.js +7 -0
package/dist/application/index.d.ts +2 -0
package/dist/application/index.js +2 -0
package/dist/application/usecase.d.ts +3 -0
package/dist/application/usecase.js +1 -0
package/dist/application/usecases.d.ts +114 -0
package/dist/application/usecases.js +201 -0
package/dist/audit/index.js +1 -1
package/dist/audit/transport.d.ts +1 -1
package/dist/audit/transport.js +5 -4
package/dist/audit/types.d.ts +1 -0
package/dist/constants.d.ts +17 -0
package/dist/constants.js +23 -0
package/dist/core/scope/scopeManager.js +3 -0
package/dist/core/security/ipGuard.d.ts +11 -0
package/dist/core/security/ipGuard.js +71 -3
package/dist/crawler/crawl.d.ts +4 -22
package/dist/crawler/crawl.js +4 -335
package/dist/crawler/crawler.d.ts +87 -0
package/dist/crawler/crawler.js +683 -0
package/dist/crawler/extract.d.ts +4 -1
package/dist/crawler/extract.js +7 -2
package/dist/crawler/fetcher.d.ts +2 -1
package/dist/crawler/fetcher.js +26 -11
package/dist/crawler/metricsRunner.d.ts +23 -1
package/dist/crawler/metricsRunner.js +202 -72
package/dist/crawler/normalize.d.ts +41 -0
package/dist/crawler/normalize.js +119 -3
package/dist/crawler/parser.d.ts +1 -3
package/dist/crawler/parser.js +2 -49
package/dist/crawler/resolver.d.ts +11 -0
package/dist/crawler/resolver.js +67 -0
package/dist/crawler/sitemap.d.ts +6 -0
package/dist/crawler/sitemap.js +27 -17
package/dist/crawler/trap.d.ts +5 -1
package/dist/crawler/trap.js +23 -2
package/dist/db/CrawlithDB.d.ts +110 -0
package/dist/db/CrawlithDB.js +500 -0
package/dist/db/graphLoader.js +42 -30
package/dist/db/index.d.ts +11 -0
package/dist/db/index.js +41 -29
package/dist/db/migrations.d.ts +2 -0
package/dist/db/{schema.js → migrations.js} +90 -43
package/dist/db/pluginRegistry.d.ts +9 -0
package/dist/db/pluginRegistry.js +19 -0
package/dist/db/repositories/EdgeRepository.d.ts +13 -0
package/dist/db/repositories/EdgeRepository.js +20 -0
package/dist/db/repositories/MetricsRepository.d.ts +16 -8
package/dist/db/repositories/MetricsRepository.js +28 -7
package/dist/db/repositories/PageRepository.d.ts +15 -2
package/dist/db/repositories/PageRepository.js +169 -25
package/dist/db/repositories/SiteRepository.d.ts +9 -0
package/dist/db/repositories/SiteRepository.js +13 -0
package/dist/db/repositories/SnapshotRepository.d.ts +14 -5
package/dist/db/repositories/SnapshotRepository.js +64 -5
package/dist/db/reset.d.ts +9 -0
package/dist/db/reset.js +32 -0
package/dist/db/statements.d.ts +12 -0
package/dist/db/statements.js +40 -0
package/dist/diff/compare.d.ts +0 -5
package/dist/diff/compare.js +0 -12
package/dist/diff/service.d.ts +16 -0
package/dist/diff/service.js +41 -0
package/dist/domain/index.d.ts +4 -0
package/dist/domain/index.js +4 -0
package/dist/events.d.ts +56 -0
package/dist/events.js +1 -0
package/dist/graph/graph.d.ts +36 -42
package/dist/graph/graph.js +26 -17
package/dist/graph/hits.d.ts +23 -0
package/dist/graph/hits.js +111 -0
package/dist/graph/metrics.d.ts +0 -4
package/dist/graph/metrics.js +25 -9
package/dist/graph/pagerank.d.ts +17 -4
package/dist/graph/pagerank.js +126 -91
package/dist/graph/simhash.d.ts +6 -0
package/dist/graph/simhash.js +14 -0
package/dist/index.d.ts +29 -8
package/dist/index.js +29 -8
package/dist/lock/hashKey.js +1 -1
package/dist/lock/lockManager.d.ts +5 -1
package/dist/lock/lockManager.js +38 -13
package/dist/plugin-system/plugin-cli.d.ts +10 -0
package/dist/plugin-system/plugin-cli.js +31 -0
package/dist/plugin-system/plugin-config.d.ts +16 -0
package/dist/plugin-system/plugin-config.js +36 -0
package/dist/plugin-system/plugin-loader.d.ts +17 -0
package/dist/plugin-system/plugin-loader.js +122 -0
package/dist/plugin-system/plugin-registry.d.ts +25 -0
package/dist/plugin-system/plugin-registry.js +167 -0
package/dist/plugin-system/plugin-types.d.ts +205 -0
package/dist/plugin-system/plugin-types.js +1 -0
package/dist/ports/index.d.ts +9 -0
package/dist/ports/index.js +1 -0
package/{src/report/sitegraph_template.ts → dist/report/crawl.html} +330 -81
package/dist/report/crawlExport.d.ts +3 -0
package/dist/report/{sitegraphExport.js → crawlExport.js} +3 -3
package/dist/report/crawl_template.d.ts +1 -0
package/dist/report/crawl_template.js +7 -0
package/dist/report/export.d.ts +3 -0
package/dist/report/export.js +81 -0
package/dist/report/html.js +15 -216
package/dist/report/insight.d.ts +27 -0
package/dist/report/insight.js +103 -0
package/dist/scoring/health.d.ts +56 -0
package/dist/scoring/health.js +213 -0
package/dist/utils/chalk.d.ts +6 -0
package/dist/utils/chalk.js +41 -0
package/dist/utils/secureConfig.d.ts +23 -0
package/dist/utils/secureConfig.js +128 -0
package/package.json +12 -6
package/CHANGELOG.md +0 -7
package/dist/db/schema.d.ts +0 -2
package/dist/graph/cluster.d.ts +0 -6
package/dist/graph/cluster.js +0 -173
package/dist/graph/duplicate.d.ts +0 -10
package/dist/graph/duplicate.js +0 -251
package/dist/report/sitegraphExport.d.ts +0 -3
package/dist/report/sitegraph_template.d.ts +0 -1
package/dist/report/sitegraph_template.js +0 -630
package/dist/scoring/hits.d.ts +0 -9
package/dist/scoring/hits.js +0 -111
package/src/analysis/analyze.ts +0 -548
package/src/analysis/content.ts +0 -62
package/src/analysis/images.ts +0 -28
package/src/analysis/links.ts +0 -41
package/src/analysis/scoring.ts +0 -59
package/src/analysis/seo.ts +0 -82
package/src/analysis/structuredData.ts +0 -62
package/src/audit/dns.ts +0 -49
package/src/audit/headers.ts +0 -98
package/src/audit/index.ts +0 -66
package/src/audit/scoring.ts +0 -232
package/src/audit/transport.ts +0 -258
package/src/audit/types.ts +0 -102
package/src/core/network/proxyAdapter.ts +0 -21
package/src/core/network/rateLimiter.ts +0 -39
package/src/core/network/redirectController.ts +0 -47
package/src/core/network/responseLimiter.ts +0 -34
package/src/core/network/retryPolicy.ts +0 -57
package/src/core/scope/domainFilter.ts +0 -45
package/src/core/scope/scopeManager.ts +0 -52
package/src/core/scope/subdomainPolicy.ts +0 -39
package/src/core/security/ipGuard.ts +0 -92
package/src/crawler/crawl.ts +0 -382
package/src/crawler/extract.ts +0 -34
package/src/crawler/fetcher.ts +0 -233
package/src/crawler/metricsRunner.ts +0 -124
package/src/crawler/normalize.ts +0 -108
package/src/crawler/parser.ts +0 -190
package/src/crawler/sitemap.ts +0 -73
package/src/crawler/trap.ts +0 -96
package/src/db/graphLoader.ts +0 -105
package/src/db/index.ts +0 -70
package/src/db/repositories/EdgeRepository.ts +0 -29
package/src/db/repositories/MetricsRepository.ts +0 -49
package/src/db/repositories/PageRepository.ts +0 -128
package/src/db/repositories/SiteRepository.ts +0 -32
package/src/db/repositories/SnapshotRepository.ts +0 -74
package/src/db/schema.ts +0 -177
package/src/diff/compare.ts +0 -84
package/src/graph/cluster.ts +0 -192
package/src/graph/duplicate.ts +0 -286
package/src/graph/graph.ts +0 -172
package/src/graph/metrics.ts +0 -110
package/src/graph/pagerank.ts +0 -125
package/src/graph/simhash.ts +0 -61
package/src/index.ts +0 -30
package/src/lock/hashKey.ts +0 -51
package/src/lock/lockManager.ts +0 -124
package/src/lock/pidCheck.ts +0 -13
package/src/report/html.ts +0 -227
package/src/report/sitegraphExport.ts +0 -58
package/src/scoring/hits.ts +0 -131
package/src/scoring/orphanSeverity.ts +0 -176
package/src/utils/version.ts +0 -18
package/tests/__snapshots__/orphanSeverity.test.ts.snap +0 -49
package/tests/analysis.unit.test.ts +0 -98
package/tests/analyze.integration.test.ts +0 -98
package/tests/audit/dns.test.ts +0 -31
package/tests/audit/headers.test.ts +0 -45
package/tests/audit/scoring.test.ts +0 -133
package/tests/audit/security.test.ts +0 -12
package/tests/audit/transport.test.ts +0 -112
package/tests/clustering.test.ts +0 -118
package/tests/crawler.test.ts +0 -358
package/tests/db.test.ts +0 -159
package/tests/diff.test.ts +0 -67
package/tests/duplicate.test.ts +0 -110
package/tests/fetcher.test.ts +0 -106
package/tests/fetcher_safety.test.ts +0 -85
package/tests/fixtures/analyze-crawl.json +0 -26
package/tests/hits.test.ts +0 -134
package/tests/html_report.test.ts +0 -58
package/tests/lock/lockManager.test.ts +0 -138
package/tests/metrics.test.ts +0 -196
package/tests/normalize.test.ts +0 -101
package/tests/orphanSeverity.test.ts +0 -160
package/tests/pagerank.test.ts +0 -98
package/tests/parser.test.ts +0 -117
package/tests/proxy_safety.test.ts +0 -57
package/tests/redirect_safety.test.ts +0 -73
package/tests/safety.test.ts +0 -114
package/tests/scope.test.ts +0 -66
package/tests/scoring.test.ts +0 -59
package/tests/sitemap.test.ts +0 -88
package/tests/soft404.test.ts +0 -41
package/tests/trap.test.ts +0 -39
package/tests/visualization_data.test.ts +0 -46
package/tsconfig.json +0 -11

package/{src/report/sitegraph_template.ts → dist/report/crawl.html} RENAMED Viewed

@@ -1,5 +1,6 @@
-export const SITEGRAPH_HTML = `<!DOCTYPE html>
+<!DOCTYPE html>
 <html lang="en">
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
@@ -13,81 +14,329 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
       --accent-color: #4a90e2;
       --sidebar-width: 300px;
     }
-    body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; background: var(--bg-color); color: var(--text-color); height: 100vh; display: flex; flex-direction: column; overflow: hidden; }
+    body {
+      margin: 0;
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+      background: var(--bg-color);
+      color: var(--text-color);
+      height: 100vh;
+      display: flex;
+      flex-direction: column;
+      overflow: hidden;
+    }
     /* Layout */
-    header { padding: 0 20px; background: var(--panel-bg); border-bottom: 1px solid var(--border-color); display: flex; justify-content: space-between; align-items: center; height: 60px; box-sizing: border-box; z-index: 10; }
-    main { flex: 1; display: flex; overflow: hidden; position: relative; }
-    #graph-container { flex: 1; position: relative; overflow: hidden; background: var(--bg-color); }
-    #details-panel { width: var(--sidebar-width); background: var(--panel-bg); border-left: 1px solid var(--border-color); padding: 20px; overflow-y: auto; box-sizing: border-box; display: none; flex-direction: column; gap: 15px; }
-    #details-panel.visible { display: flex; }
-    footer { padding: 5px 20px; background: var(--panel-bg); border-top: 1px solid var(--border-color); font-size: 0.8rem; text-align: center; color: #666; height: 30px; display: flex; align-items: center; justify-content: center; }
+    header {
+      padding: 0 20px;
+      background: var(--panel-bg);
+      border-bottom: 1px solid var(--border-color);
+      display: flex;
+      justify-content: space-between;
+      align-items: center;
+      height: 60px;
+      box-sizing: border-box;
+      z-index: 10;
+    }
+    main {
+      flex: 1;
+      display: flex;
+      overflow: hidden;
+      position: relative;
+    }
+    #graph-container {
+      flex: 1;
+      position: relative;
+      overflow: hidden;
+      background: var(--bg-color);
+    }
+    #details-panel {
+      width: var(--sidebar-width);
+      background: var(--panel-bg);
+      border-left: 1px solid var(--border-color);
+      padding: 20px;
+      overflow-y: auto;
+      box-sizing: border-box;
+      display: none;
+      flex-direction: column;
+      gap: 15px;
+    }
+    #details-panel.visible {
+      display: flex;
+    }
+    footer {
+      padding: 5px 20px;
+      background: var(--panel-bg);
+      border-top: 1px solid var(--border-color);
+      font-size: 0.8rem;
+      text-align: center;
+      color: #666;
+      height: 30px;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+    }
     /* Header Components */
-    .brand { font-weight: bold; font-size: 1.2rem; display: flex; align-items: center; gap: 10px; }
-    .brand span { color: var(--accent-color); }
-    #metrics-summary { font-size: 0.9rem; color: #aaa; display: flex; gap: 20px; }
-    .metric { display: flex; flex-direction: column; align-items: center; line-height: 1.1; }
-    .metric-value { font-weight: bold; color: var(--text-color); }
-    .metric-label { font-size: 0.7rem; }
-    #controls { display: flex; gap: 10px; align-items: center; }
-    .btn-group { display: flex; background: #333; border-radius: 4px; overflow: hidden; }
-    button { background: transparent; color: #aaa; border: none; padding: 6px 12px; cursor: pointer; font-size: 0.85rem; transition: all 0.2s; }
-    button:hover { color: white; background: rgba(255,255,255,0.1); }
-    button.active { background: var(--accent-color); color: white; }
+    .brand {
+      font-weight: bold;
+      font-size: 1.2rem;
+      display: flex;
+      align-items: center;
+      gap: 10px;
+    }
+    .brand span {
+      color: var(--accent-color);
+    }
+    #metrics-summary {
+      font-size: 0.9rem;
+      color: #aaa;
+      display: flex;
+      gap: 20px;
+    }
+    .metric {
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      line-height: 1.1;
+    }
+    .metric-value {
+      font-weight: bold;
+      color: var(--text-color);
+    }
+    .metric-label {
+      font-size: 0.7rem;
+    }
+    #controls {
+      display: flex;
+      gap: 10px;
+      align-items: center;
+    }
+    .btn-group {
+      display: flex;
+      background: #333;
+      border-radius: 4px;
+      overflow: hidden;
+    }
+    button {
+      background: transparent;
+      color: #aaa;
+      border: none;
+      padding: 6px 12px;
+      cursor: pointer;
+      font-size: 0.85rem;
+      transition: all 0.2s;
+    }
+    button:hover {
+      color: white;
+      background: rgba(255, 255, 255, 0.1);
+    }
+    button.active {
+      background: var(--accent-color);
+      color: white;
+    }
     /* Search */
-    #search-container { position: absolute; top: 15px; left: 15px; z-index: 5; }
-    #search-input { background: rgba(30,30,30,0.9); border: 1px solid #444; color: white; padding: 8px 12px; border-radius: 20px; width: 200px; outline: none; transition: width 0.3s; }
-    #search-input:focus { width: 280px; border-color: var(--accent-color); }
+    #search-container {
+      position: absolute;
+      top: 15px;
+      left: 15px;
+      z-index: 5;
+    }
+    #search-input {
+      background: rgba(30, 30, 30, 0.9);
+      border: 1px solid #444;
+      color: white;
+      padding: 8px 12px;
+      border-radius: 20px;
+      width: 200px;
+      outline: none;
+      transition: width 0.3s;
+    }
+    #search-input:focus {
+      width: 280px;
+      border-color: var(--accent-color);
+    }
     /* Graph */
-    svg { width: 100%; height: 100%; display: block; }
-    .node { cursor: pointer; transition: stroke-width 0.1s; }
-    .link { stroke: #555; stroke-opacity: 0.3; fill: none; pointer-events: none; }
+    svg {
+      width: 100%;
+      height: 100%;
+      display: block;
+    }
+    .node {
+      cursor: pointer;
+      transition: stroke-width 0.1s;
+    }
+    .link {
+      stroke: #555;
+      stroke-opacity: 0.3;
+      fill: none;
+      pointer-events: none;
+    }
     /* Interaction States */
-    .node.highlight { stroke: #fff; stroke-width: 2px; }
-    .link.highlight { stroke-opacity: 0.8; stroke: #999; }
-    .node.faded { opacity: 0.1; }
-    .link.faded { opacity: 0.05; }
+    .node.highlight {
+      stroke: #fff;
+      stroke-width: 2px;
+    }
+    .link.highlight {
+      stroke-opacity: 0.8;
+      stroke: #999;
+    }
+    .node.faded {
+      opacity: 0.1;
+    }
+    .link.faded {
+      opacity: 0.05;
+    }
     /* Details Panel Content */
-    .detail-section { border-bottom: 1px solid #333; padding-bottom: 10px; }
-    .detail-section:last-child { border-bottom: none; }
-    .detail-label { font-size: 0.75rem; color: #888; text-transform: uppercase; letter-spacing: 0.5px; margin-bottom: 4px; }
-    .detail-value { font-size: 0.95rem; word-break: break-all; }
-    .detail-list { list-style: none; padding: 0; margin: 0; max-height: 150px; overflow-y: auto; font-size: 0.85rem; }
-    .detail-list li { padding: 4px 0; border-bottom: 1px solid #2a2a2a; }
-    .detail-list a { color: var(--accent-color); text-decoration: none; }
-    .detail-list a:hover { text-decoration: underline; }
-    .status-badge { display: inline-block; padding: 2px 6px; border-radius: 3px; font-size: 0.75rem; font-weight: bold; margin-top: 5px; }
-    .status-ok { background: #2e7d32; color: white; }
-    .status-warn { background: #f9a825; color: black; }
-    .status-error { background: #c62828; color: white; }
+    .detail-section {
+      border-bottom: 1px solid #333;
+      padding-bottom: 10px;
+    }
+    .detail-section:last-child {
+      border-bottom: none;
+    }
+    .detail-label {
+      font-size: 0.75rem;
+      color: #888;
+      text-transform: uppercase;
+      letter-spacing: 0.5px;
+      margin-bottom: 4px;
+    }
+    .detail-value {
+      font-size: 0.95rem;
+      word-break: break-all;
+    }
+    .detail-list {
+      list-style: none;
+      padding: 0;
+      margin: 0;
+      max-height: 150px;
+      overflow-y: auto;
+      font-size: 0.85rem;
+    }
+    .detail-list li {
+      padding: 4px 0;
+      border-bottom: 1px solid #2a2a2a;
+    }
+    .detail-list a {
+      color: var(--accent-color);
+      text-decoration: none;
+    }
+    .detail-list a:hover {
+      text-decoration: underline;
+    }
+    .status-badge {
+      display: inline-block;
+      padding: 2px 6px;
+      border-radius: 3px;
+      font-size: 0.75rem;
+      font-weight: bold;
+      margin-top: 5px;
+    }
+    .status-ok {
+      background: #2e7d32;
+      color: white;
+    }
+    .status-warn {
+      background: #f9a825;
+      color: black;
+    }
+    .status-error {
+      background: #c62828;
+      color: white;
+    }
     /* Tooltip */
-    #tooltip { position: absolute; background: rgba(20,20,20,0.95); color: white; padding: 10px; border-radius: 6px; pointer-events: none; font-size: 12px; z-index: 100; box-shadow: 0 4px 15px rgba(0,0,0,0.5); border: 1px solid #444; display: none; transform: translate(-50%, -100%); margin-top: -10px; white-space: nowrap; }
+    #tooltip {
+      position: absolute;
+      background: rgba(20, 20, 20, 0.95);
+      color: white;
+      padding: 10px;
+      border-radius: 6px;
+      pointer-events: none;
+      font-size: 12px;
+      z-index: 100;
+      box-shadow: 0 4px 15px rgba(0, 0, 0, 0.5);
+      border: 1px solid #444;
+      display: none;
+      transform: translate(-50%, -100%);
+      margin-top: -10px;
+      white-space: nowrap;
+    }
     /* Responsive Sidebar */
     @media (max-width: 768px) {
-      #details-panel { position: absolute; right: 0; top: 0; bottom: 0; z-index: 20; box-shadow: -5px 0 15px rgba(0,0,0,0.5); transform: translateX(100%); transition: transform 0.3s ease; }
-      #details-panel.visible { transform: translateX(0); }
-      #metrics-summary { display: none; }
+      #details-panel {
+        position: absolute;
+        right: 0;
+        top: 0;
+        bottom: 0;
+        z-index: 20;
+        box-shadow: -5px 0 15px rgba(0, 0, 0, 0.5);
+        transform: translateX(100%);
+        transition: transform 0.3s ease;
+      }
+      #details-panel.visible {
+        transform: translateX(0);
+      }
+      #metrics-summary {
+        display: none;
+      }
     }
   </style>
 </head>
 <body>
   <header>
-    <div class="brand"><span>Crawlith</span> SiteGraph</div>
+    <div class="brand"><span>Crawlith</span> Crawl</div>
     <div id="metrics-summary">
       <div class="metric"><span class="metric-value" id="m-pages">-</span><span class="metric-label">Pages</span></div>
-      <div class="metric"><span class="metric-value" id="m-depth">-</span><span class="metric-label">Max Depth</span></div>
-      <div class="metric"><span class="metric-value" id="m-eff">-</span><span class="metric-label">Efficiency</span></div>
-      <div class="metric"><span class="metric-value" id="m-orphan">-</span><span class="metric-label">Orphans</span></div>
+      <div class="metric"><span class="metric-value" id="m-depth">-</span><span class="metric-label">Max Depth</span>
+      </div>
+      <div class="metric"><span class="metric-value" id="m-eff">-</span><span class="metric-label">Efficiency</span>
+      </div>
+      <div class="metric"><span class="metric-value" id="m-orphan">-</span><span class="metric-label">Orphans</span>
+      </div>
     </div>
     <div id="controls">
@@ -129,7 +378,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
       </div>
       <div class="detail-section">
         <div class="detail-label">In-links (<span id="d-in-count">0</span>)</div>
-         <!-- List could be populated here if we had the reverse index, for now just count -->
+        <!-- List could be populated here if we had the reverse index, for now just count -->
       </div>
       <div class="detail-section">
         <div class="detail-label">Out-links (<span id="d-out-count">0</span>)</div>
@@ -187,22 +436,22 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
         // 2. Fallback to fetching JSON files (for web server usage)
         if (!graphData || !metricsData) {
-            try {
-                const [graphRes, metricsRes] = await Promise.all([
-                    fetch('graph.json'),
-                    fetch('metrics.json')
-                ]);
-                if (graphRes.ok && metricsRes.ok) {
-                    graphData = await graphRes.json();
-                    metricsData = await metricsRes.json();
-                }
-            } catch (e) {
-                console.warn("Fetch failed, possibly due to CORS or missing files.", e);
+          try {
+            const [graphRes, metricsRes] = await Promise.all([
+              fetch('graph.json'),
+              fetch('metrics.json')
+            ]);
+            if (graphRes.ok && metricsRes.ok) {
+              graphData = await graphRes.json();
+              metricsData = await metricsRes.json();
             }
+          } catch (e) {
+            console.warn("Fetch failed, possibly due to CORS or missing files.", e);
+          }
         }
         if (!graphData || !metricsData) {
-            throw new Error("No data available. Ensure graph.json exists or data is injected.");
+          throw new Error("No data available. Ensure graph.json exists or data is injected.");
         }
         state.metrics = metricsData;
@@ -323,12 +572,12 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
         const depthSpacing = height / (state.maxDepth + 2);
         // Hierarchical: Nodes pushed to Y levels based on depth
         state.simulation.force("y", d3.forceY(d => {
-           return (d.depth * depthSpacing) - (height/2) + 50; // Offset to start from top
+          return (d.depth * depthSpacing) - (height / 2) + 50; // Offset to start from top
         }).strength(1));
         // We rely on "center" force to keep X centered, but maybe add weak forceX?
         // Let's add weak forceX to prevent wide spread
         state.simulation.force("x", d3.forceX(0).strength(0.05));
-        state.simulation.force("center", d3.forceCenter(width/2, height/2)); // Recenter
+        state.simulation.force("center", d3.forceCenter(width / 2, height / 2)); // Recenter
       } else if (mode === 'radial') {
         const maxRadius = Math.min(width, height) / 2 - 50;
@@ -401,7 +650,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
         });
       svg.call(state.zoom)
-         .call(state.zoom.transform, d3.zoomIdentity.translate(state.width/2, state.height/2).scale(0.8).translate(-state.width/2, -state.height/2)); // Initial zoom out
+        .call(state.zoom.transform, d3.zoomIdentity.translate(state.width / 2, state.height / 2).scale(0.8).translate(-state.width / 2, -state.height / 2)); // Initial zoom out
     }
     function ticked() {
@@ -431,18 +680,18 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
       });
       // Layout Toggle
-      d3.select("#btn-hierarchical").on("click", function() {
+      d3.select("#btn-hierarchical").on("click", function () {
         setMode('hierarchical', this);
       });
-      d3.select("#btn-radial").on("click", function() {
+      d3.select("#btn-radial").on("click", function () {
         setMode('radial', this);
       });
       // Authority Toggle
-      d3.select("#btn-auth-pagerank").on("click", function() {
+      d3.select("#btn-auth-pagerank").on("click", function () {
         setAuthorityMode('pagerank', this);
       });
-      d3.select("#btn-auth-structural").on("click", function() {
+      d3.select("#btn-auth-structural").on("click", function () {
         setAuthorityMode('structural', this);
       });
     }
@@ -508,7 +757,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
     function showTooltip(event, d) {
       // If we are transforming the container, we need to map coordinates correctly or just use pageX/Y
       tooltip.style("display", "block")
-        .html(\`<strong>\${new URL(d.url).pathname}</strong><br>Auth: \${(d.authority * 10).toFixed(1)}\`)
+        .html(`<strong>${new URL(d.url).pathname}</strong><br>Auth: ${(d.authority * 10).toFixed(1)}`)
         .style("left", (event.pageX) + "px")
         .style("top", (event.pageY - 10) + "px");
     }
@@ -526,8 +775,8 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
       authContainer.html("");
       const prVal = (d.pageRankAuthority * 100).toFixed(1);
       const structVal = d.structuralAuthority.toFixed(3);
-      authContainer.append("div").html(\`PR: <strong>\${prVal}</strong>\`);
-      authContainer.append("div").style("color", "#888").style("font-size", "0.8em").text(\`In-Degree: \${structVal}\`);
+      authContainer.append("div").html(`PR: <strong>${prVal}</strong>`);
+      authContainer.append("div").style("color", "#888").style("font-size", "0.8em").text(`In-Degree: ${structVal}`);
       d3.select("#d-in-count").text(d.inLinks);
       d3.select("#d-out-count").text(d.outLinks);
@@ -552,7 +801,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
             .text(new URL(target.url).pathname);
         });
         if (adj.out.length > 50) {
-            list.append("li").text(\`...and \${adj.out.length - 50} more\`);
+          list.append("li").text(`...and ${adj.out.length - 50} more`);
         }
       } else {
         list.append("li").text("No outgoing links");
@@ -572,7 +821,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
             selectNode(found);
             // Center view on node
             const transform = d3.zoomIdentity
-              .translate(state.width/2, state.height/2)
+              .translate(state.width / 2, state.height / 2)
               .scale(2)
               .translate(-found.x, -found.y);
@@ -589,7 +838,7 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
         state.height = height;
         state.simulation.force("center", d3.forceCenter(width / 2, height / 2));
         if (state.mode === 'hierarchical') {
-            // Re-evaluate Y force if needed, but usually center is enough
+          // Re-evaluate Y force if needed, but usually center is enough
         }
         state.simulation.alpha(0.3).restart();
       });
@@ -620,11 +869,11 @@ export const SITEGRAPH_HTML = `<!DOCTYPE html>
     // Start
     if (document.readyState === 'loading') {
-        document.addEventListener('DOMContentLoaded', init);
+      document.addEventListener('DOMContentLoaded', init);
     } else {
-        init();
+      init();
     }
   </script>
 </body>
-</html>
-`;
+</html>

package/dist/report/crawlExport.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+export declare function renderCrawlCsvNodes(graphData: any): string;
+export declare function renderCrawlCsvEdges(graphData: any): string;
+export declare function renderCrawlMarkdown(url: string, graphData: any, metrics: any, graph: any): string;

package/dist/report/{sitegraphExport.js → crawlExport.js} RENAMED Viewed

@@ -1,4 +1,4 @@
-export function renderSitegraphCsvNodes(graphData) {
+export function renderCrawlCsvNodes(graphData) {
     const nodeHeaders = ['URL', 'Depth', 'Status', 'InboundLinks', 'OutboundLinks', 'PageRankScore'];
     const nodeRows = graphData.nodes.map((n) => {
         const outbound = graphData.edges.filter((e) => e.source === n.url).length;
@@ -8,12 +8,12 @@ export function renderSitegraphCsvNodes(graphData) {
     });
     return [nodeHeaders.join(','), ...nodeRows].join('\n');
 }
-export function renderSitegraphCsvEdges(graphData) {
+export function renderCrawlCsvEdges(graphData) {
     const edgeHeaders = ['Source', 'Target', 'Weight'];
     const edgeRows = graphData.edges.map((e) => [e.source, e.target, e.weight].join(','));
     return [edgeHeaders.join(','), ...edgeRows].join('\n');
 }
-export function renderSitegraphMarkdown(url, graphData, metrics, graph) {
+export function renderCrawlMarkdown(url, graphData, metrics, graph) {
     const md = [
         `# Crawlith Crawl Summary - ${url}`,
         '',

package/dist/report/crawl_template.d.ts ADDED Viewed

	@@ -0,0 +1 @@
1	+ export declare const Crawl_HTML: string;

package/dist/report/crawl_template.js ADDED Viewed

@@ -0,0 +1,7 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const templatePath = path.join(__dirname, 'crawl.html');
+export const Crawl_HTML = fs.readFileSync(templatePath, 'utf-8');

package/dist/report/export.d.ts ADDED Viewed

@@ -0,0 +1,3 @@
+export declare function parseExportFormats(exportOption: string | boolean | undefined): string[];
+export declare function runCrawlExports(formats: string[], outputDir: string, url: string, graphData: any, metrics: any, graphObj: any, report?: any): Promise<void>;
+export declare function runAnalysisExports(formats: string[], outputDir: string, result: any, isLive: boolean): Promise<void>;

package/dist/report/export.js ADDED Viewed

@@ -0,0 +1,81 @@
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import chalk from '../utils/chalk.js';
+import { generateHtml, } from './html.js';
+import { renderCrawlMarkdown, renderCrawlCsvNodes, renderCrawlCsvEdges } from './crawlExport.js';
+import { renderAnalysisHtml, renderAnalysisMarkdown, renderAnalysisCsv } from '../analysis/analyze.js';
+export function parseExportFormats(exportOption) {
+    if (exportOption === undefined || exportOption === false)
+        return [];
+    if (exportOption === true)
+        return ['json'];
+    return exportOption.split(',').map(s => s.trim().toLowerCase()).filter(Boolean);
+}
+export async function runCrawlExports(formats, outputDir, url, graphData, metrics, graphObj, report) {
+    if (formats.length === 0)
+        return;
+    await fs.mkdir(outputDir, { recursive: true });
+    if (formats.includes('json')) {
+        await fs.writeFile(path.join(outputDir, 'graph.json'), JSON.stringify(graphData, null, 2));
+        await fs.writeFile(path.join(outputDir, 'metrics.json'), JSON.stringify(metrics, null, 2));
+        if (report) {
+            await fs.writeFile(path.join(outputDir, 'report.json'), JSON.stringify(report, null, 2));
+        }
+        console.log(chalk.green(`JSON exports saved to ${outputDir} (graph.json, metrics.json${report ? ', report.json' : ''})`));
+    }
+    if (formats.includes('html')) {
+        const html = generateHtml(graphData, metrics);
+        await fs.writeFile(path.join(outputDir, 'graph.html'), html);
+        console.log(chalk.green(`HTML report saved to ${path.join(outputDir, 'graph.html')}`));
+    }
+    if (formats.includes('visualize')) {
+        const CrawlHtml = generateHtml(graphData, metrics);
+        await fs.writeFile(path.join(outputDir, 'crawl.html'), CrawlHtml);
+        console.log(chalk.green(`Visualization saved to ${path.join(outputDir, 'crawl.html')}`));
+    }
+    if (formats.includes('csv')) {
+        await fs.writeFile(path.join(outputDir, 'nodes.csv'), renderCrawlCsvNodes(graphData));
+        await fs.writeFile(path.join(outputDir, 'edges.csv'), renderCrawlCsvEdges(graphData));
+        console.log(chalk.green(`CSV exports saved to ${outputDir} (nodes.csv, edges.csv)`));
+    }
+    if (formats.includes('markdown')) {
+        const md = renderCrawlMarkdown(url, graphData, metrics, graphObj);
+        await fs.writeFile(path.join(outputDir, 'summary.md'), md);
+        console.log(chalk.green(`Markdown summary saved to ${path.join(outputDir, 'summary.md')}`));
+        if (report && report.plugins) {
+            for (const [pluginName, pluginData] of Object.entries(report.plugins)) {
+                // Ensure Exporter remains generic without plugin-specific logical branches
+                const serialized = JSON.stringify(pluginData, null, 2);
+                const pluginMd = `\n## Plugin: ${pluginName}\n\n\`\`\`json\n${serialized}\n\`\`\`\n`;
+                await fs.appendFile(path.join(outputDir, 'summary.md'), pluginMd);
+            }
+        }
+    }
+}
+export async function runAnalysisExports(formats, outputDir, result, isLive) {
+    if (formats.length === 0)
+        return;
+    await fs.mkdir(outputDir, { recursive: true });
+    if (formats.includes('json')) {
+        await fs.writeFile(path.join(outputDir, 'analysis.json'), JSON.stringify(result, null, 2));
+        console.log(chalk.green(`JSON export saved to ${path.join(outputDir, 'analysis.json')}`));
+    }
+    if (formats.includes('html')) {
+        const html = renderAnalysisHtml(result);
+        const filename = isLive ? 'page.html' : 'analysis.html';
+        await fs.writeFile(path.join(outputDir, filename), html, 'utf-8');
+        console.log(chalk.green(`HTML report saved to ${path.join(outputDir, filename)}`));
+    }
+    if (formats.includes('markdown')) {
+        const markdown = renderAnalysisMarkdown(result);
+        const filename = isLive ? 'analysis.md' : 'analysis.md';
+        await fs.writeFile(path.join(outputDir, filename), markdown, 'utf-8');
+        console.log(chalk.green(`Markdown report saved to ${path.join(outputDir, filename)}`));
+    }
+    if (formats.includes('csv')) {
+        const csv = renderAnalysisCsv(result);
+        const filename = isLive ? 'analysis.csv' : 'analysis.csv';
+        await fs.writeFile(path.join(outputDir, filename), csv, 'utf-8');
+        console.log(chalk.green(`CSV export saved to ${path.join(outputDir, filename)}`));
+    }
+}