@crawlith/core 0.1.1 → 0.1.2

package/dist/graph/duplicate.js

@@ -1,302 +0,0 @@
-import { Graph } from './graph.js';
-import { SimHash } from './simhash.js';
-/**
- * Detects exact and near duplicates, identifies canonical conflicts,
- * and performs non-destructive collapse of edges.
- */
-export function detectDuplicates(graph, options = {}) {
-    const collapse = options.collapse !== false; // Default to true
-    const threshold = options.simhashThreshold ?? 3;
-    const nodes = graph.getNodes();
-    let clusterCounter = 1;
-    // Phase 1 & 2: Exact Duplicate Detection
-    const { exactClusters, nearCandidates, nextId: nextId1 } = findExactDuplicates(nodes, clusterCounter);
-    clusterCounter = nextId1;
-    // Phase 3: Near Duplicate Detection
-    const { nearClusters } = findNearDuplicates(nearCandidates, threshold, clusterCounter);
-    const allClusters = [...exactClusters, ...nearClusters];
-    // Phase 4, 5, 6: Process Clusters (Template-Heavy, Canonical, Representative)
-    processClusters(allClusters, graph, collapse);
-    // Final Edge Transfer if Collapsing
-    if (collapse) {
-        collapseEdges(graph);
-    }
-}
-function findExactDuplicates(nodes, startId) {
-    const exactMap = groupNodesByContentHash(nodes);
-    return createExactClusters(exactMap, startId);
-}
-function groupNodesByContentHash(nodes) {
-    const exactMap = new Map();
-    for (const node of nodes) {
-        if (!node.contentHash || node.status !== 200)
-            continue;
-        let arr = exactMap.get(node.contentHash);
-        if (!arr) {
-            arr = [];
-            exactMap.set(node.contentHash, arr);
-        }
-        arr.push(node);
-    }
-    return exactMap;
-}
-function createExactClusters(exactMap, startId) {
-    const exactClusters = [];
-    const nearCandidates = [];
-    let clusterCounter = startId;
-    for (const [_hash, group] of exactMap.entries()) {
-        if (group.length > 1) {
-            const id = `cluster_exact_${clusterCounter++}`;
-            exactClusters.push({ id, type: 'exact', nodes: group });
-            for (const n of group) {
-                n.duplicateClusterId = id;
-                n.duplicateType = 'exact';
-            }
-        }
-        else {
-            nearCandidates.push(group[0]);
-        }
-    }
-    return { exactClusters, nearCandidates, nextId: clusterCounter };
-}
-function findNearDuplicates(candidates, threshold, startId) {
-    const { bandsMaps, simhashes } = buildSimHashBuckets(candidates);
-    const { parent, involvedIndices } = findConnectedComponents(bandsMaps, simhashes, candidates.length, threshold);
-    return extractClusters(parent, involvedIndices, candidates, startId);
-}
-function buildSimHashBuckets(candidates) {
-    const n = candidates.length;
-    const simhashes = new BigUint64Array(n);
-    const validIndices = [];
-    for (let i = 0; i < n; i++) {
-        if (candidates[i].simhash) {
-            simhashes[i] = BigInt(candidates[i].simhash);
-            validIndices.push(i);
-        }
-    }
-    const bandsMaps = Array.from({ length: SimHash.BANDS }, () => new Map());
-    for (const idx of validIndices) {
-        const bands = SimHash.getBands(simhashes[idx]);
-        for (let b = 0; b < SimHash.BANDS; b++) {
-            let arr = bandsMaps[b].get(bands[b]);
-            if (!arr) {
-                arr = [];
-                bandsMaps[b].set(bands[b], arr);
-            }
-            arr.push(idx);
-        }
-    }
-    return { bandsMaps, simhashes, validIndices };
-}
-function findConnectedComponents(bandsMaps, simhashes, n, threshold) {
-    // Union-Find Arrays (Integer-based)
-    const parent = new Uint32Array(n);
-    const rank = new Uint8Array(n);
-    for (let i = 0; i < n; i++) {
-        parent[i] = i;
-        rank[i] = 0;
-    }
-    function find(i) {
-        let root = i;
-        while (parent[root] !== root) {
-            root = parent[root];
-        }
-        let curr = i;
-        while (curr !== root) {
-            const next = parent[curr];
-            parent[curr] = root;
-            curr = next;
-        }
-        return root;
-    }
-    function union(i, j) {
-        const rootI = find(i);
-        const rootJ = find(j);
-        if (rootI !== rootJ) {
-            const rankI = rank[rootI];
-            const rankJ = rank[rootJ];
-            if (rankI < rankJ) {
-                parent[rootI] = rootJ;
-            }
-            else if (rankI > rankJ) {
-                parent[rootJ] = rootI;
-            }
-            else {
-                parent[rootJ] = rootI;
-                rank[rootI]++;
-            }
-        }
-    }
-    const involvedIndices = new Set();
-    for (let b = 0; b < SimHash.BANDS; b++) {
-        for (const bucketIndices of bandsMaps[b].values()) {
-            if (bucketIndices.length < 2)
-                continue;
-            for (let j = 0; j < bucketIndices.length; j++) {
-                for (let k = j + 1; k < bucketIndices.length; k++) {
-                    const idx1 = bucketIndices[j];
-                    const idx2 = bucketIndices[k];
-                    const root1 = find(idx1);
-                    const root2 = find(idx2);
-                    if (root1 === root2)
-                        continue; // Already connected, skip expensive distance check
-                    const dist = SimHash.hammingDistance(simhashes[idx1], simhashes[idx2]);
-                    if (dist <= threshold) {
-                        union(root1, root2);
-                        involvedIndices.add(idx1);
-                        involvedIndices.add(idx2);
-                    }
-                }
-            }
-        }
-    }
-    return { parent, involvedIndices };
-}
-function extractClusters(parent, involvedIndices, candidates, startId) {
-    const nearClusters = [];
-    let clusterCounter = startId;
-    function find(i) {
-        let root = i;
-        while (parent[root] !== root) {
-            root = parent[root];
-        }
-        let curr = i;
-        while (curr !== root) {
-            const next = parent[curr];
-            parent[curr] = root;
-            curr = next;
-        }
-        return root;
-    }
-    // Compile clusters
-    const clusterMap = new Map();
-    for (const idx of involvedIndices) {
-        const root = find(idx);
-        let group = clusterMap.get(root);
-        if (!group) {
-            group = [];
-            clusterMap.set(root, group);
-        }
-        group.push(idx);
-    }
-    for (const groupIndices of clusterMap.values()) {
-        if (groupIndices.length > 1) {
-            const id = `cluster_near_${clusterCounter++}`;
-            const groupNodes = groupIndices.map(idx => candidates[idx]);
-            nearClusters.push({ id, type: 'near', nodes: groupNodes });
-            for (const n of groupNodes) {
-                n.duplicateClusterId = id;
-                n.duplicateType = 'near';
-            }
-        }
-    }
-    return { nearClusters, nextId: clusterCounter };
-}
-function processClusters(clusters, graph, collapse) {
-    for (const cluster of clusters) {
-        processSingleCluster(cluster, graph, collapse);
-    }
-}
-function processSingleCluster(cluster, graph, collapse) {
-    checkTemplateHeavy(cluster);
-    cluster.severity = calculateSeverity(cluster);
-    const representative = selectRepresentative(cluster);
-    cluster.representative = representative.url;
-    applyClusterToGraph(cluster, representative, graph, collapse);
-}
-function checkTemplateHeavy(cluster) {
-    const avgRatio = cluster.nodes.reduce((sum, n) => sum + (n.uniqueTokenRatio || 0), 0) / cluster.nodes.length;
-    if (avgRatio < 0.3) {
-        cluster.type = 'template_heavy';
-        cluster.nodes.forEach(n => n.duplicateType = 'template_heavy');
-    }
-}
-function calculateSeverity(cluster) {
-    const canonicals = new Set();
-    let hasMissing = false;
-    for (const n of cluster.nodes) {
-        if (!n.canonical)
-            hasMissing = true;
-        else
-            canonicals.add(n.canonical);
-    }
-    if (hasMissing || canonicals.size > 1) {
-        return 'high';
-    }
-    else if (cluster.type === 'near') {
-        return 'medium';
-    }
-    else {
-        return 'low';
-    }
-}
-function selectRepresentative(cluster) {
-    const urlsInCluster = new Set(cluster.nodes.map(n => n.url));
-    const validCanonicals = cluster.nodes.filter(n => n.canonical && urlsInCluster.has(n.canonical) && n.url === n.canonical);
-    if (validCanonicals.length > 0) {
-        return validCanonicals[0];
-    }
-    return cluster.nodes.reduce((best, current) => {
-        if (current.inLinks > best.inLinks)
-            return current;
-        if (current.inLinks < best.inLinks)
-            return best;
-        if (current.url.length < best.url.length)
-            return current;
-        return best;
-    });
-}
-function applyClusterToGraph(cluster, representative, graph, collapse) {
-    cluster.nodes.forEach(n => {
-        n.isClusterPrimary = n.url === representative.url;
-        n.isCollapsed = false;
-        n.collapseInto = undefined;
-    });
-    graph.duplicateClusters.push({
-        id: cluster.id,
-        type: cluster.type,
-        size: cluster.nodes.length,
-        representative: representative.url,
-        severity: cluster.severity
-    });
-    if (collapse) {
-        for (const n of cluster.nodes) {
-            if (n.url !== representative.url) {
-                n.isCollapsed = true;
-                n.collapseInto = representative.url;
-            }
-        }
-    }
-}
-function collapseEdges(graph) {
-    const edges = graph.getEdges();
-    const updatedEdges = new Map();
-    for (const edge of edges) {
-        const sourceNode = graph.nodes.get(edge.source);
-        const targetNode = graph.nodes.get(edge.target);
-        if (!sourceNode || !targetNode)
-            continue;
-        const actualSource = edge.source;
-        const actualTarget = targetNode.isCollapsed && targetNode.collapseInto ? targetNode.collapseInto : edge.target;
-        if (actualSource === actualTarget)
-            continue;
-        const edgeKey = Graph.getEdgeKey(actualSource, actualTarget);
-        const existingWeight = updatedEdges.get(edgeKey) || 0;
-        updatedEdges.set(edgeKey, Math.max(existingWeight, edge.weight));
-    }
-    graph.edges = updatedEdges;
-    // Re-calculate inLinks and outLinks based on collapsed edges
-    for (const node of graph.getNodes()) {
-        node.inLinks = 0;
-        node.outLinks = 0;
-    }
-    for (const [edgeKey, _weight] of updatedEdges.entries()) {
-        const { source: src, target: tgt } = Graph.parseEdgeKey(edgeKey);
-        const sn = graph.nodes.get(src);
-        const tn = graph.nodes.get(tgt);
-        if (sn)
-            sn.outLinks++;
-        if (tn)
-            tn.inLinks++;
-    }
-}

package/dist/scoring/hits.d.ts

@@ -1,10 +0,0 @@
-import { Graph } from '../graph/graph.js';
-export interface HITSOptions {
-    iterations?: number;
-}
-/**
- * Computes Hub and Authority scores using the HITS algorithm.
- * Operates purely on the internal link graph.
- * Optimized for performance using array-based adjacency lists.
- */
-export declare function computeHITS(graph: Graph, options?: HITSOptions): void;

package/dist/scoring/hits.js

@@ -1,131 +0,0 @@
-/**
- * Computes Hub and Authority scores using the HITS algorithm.
- * Operates purely on the internal link graph.
- * Optimized for performance using array-based adjacency lists.
- */
-export function computeHITS(graph, options = {}) {
-    const iterations = options.iterations || 20;
-    const nodes = graph.getNodes();
-    // 1. Filter eligible nodes
-    // Eligibility: status 200 (crawled) or status 0 (discovered)
-    // Non-redirect, not noindex (if known), non-external
-    const eligibleNodes = nodes.filter(n => (n.status === 200 || n.status === 0) &&
-        (!n.redirectChain || n.redirectChain.length === 0) &&
-        !n.noindex);
-    const N = eligibleNodes.length;
-    if (N === 0)
-        return;
-    // Map URL to Index for O(1) access
-    const urlToIndex = new Map();
-    for (let i = 0; i < N; i++) {
-        urlToIndex.set(eligibleNodes[i].url, i);
-    }
-    // Build Adjacency Lists (Indices)
-    // incoming[i] = list of { sourceIndex, weight }
-    // outgoing[i] = list of { targetIndex, weight }
-    const incoming = new Array(N).fill(null).map(() => []);
-    const outgoing = new Array(N).fill(null).map(() => []);
-    const allEdges = graph.getEdges();
-    for (const edge of allEdges) {
-        if (edge.source === edge.target)
-            continue;
-        const sourceIndex = urlToIndex.get(edge.source);
-        const targetIndex = urlToIndex.get(edge.target);
-        if (sourceIndex !== undefined && targetIndex !== undefined) {
-            incoming[targetIndex].push({ sourceIndex, weight: edge.weight });
-            outgoing[sourceIndex].push({ targetIndex, weight: edge.weight });
-        }
-    }
-    // Initialize Scores
-    const authScores = new Float64Array(N).fill(1.0);
-    const hubScores = new Float64Array(N).fill(1.0);
-    // 2. Iteration
-    for (let iter = 0; iter < iterations; iter++) {
-        // Update Authorities
-        let normAuth = 0;
-        for (let i = 0; i < N; i++) {
-            const inLinks = incoming[i];
-            let newAuth = 0;
-            for (let j = 0; j < inLinks.length; j++) {
-                const link = inLinks[j];
-                newAuth += hubScores[link.sourceIndex] * link.weight;
-            }
-            authScores[i] = newAuth;
-            normAuth += newAuth * newAuth;
-        }
-        // Normalize Authorities (L2 norm)
-        normAuth = Math.sqrt(normAuth);
-        if (normAuth > 0) {
-            for (let i = 0; i < N; i++) {
-                authScores[i] /= normAuth;
-            }
-        }
-        // Update Hubs
-        let normHub = 0;
-        for (let i = 0; i < N; i++) {
-            const outLinks = outgoing[i];
-            let newHub = 0;
-            for (let j = 0; j < outLinks.length; j++) {
-                const link = outLinks[j];
-                newHub += authScores[link.targetIndex] * link.weight;
-            }
-            hubScores[i] = newHub;
-            normHub += newHub * newHub;
-        }
-        // Normalize Hubs (L2 norm)
-        normHub = Math.sqrt(normHub);
-        if (normHub > 0) {
-            for (let i = 0; i < N; i++) {
-                hubScores[i] /= normHub;
-            }
-        }
-    }
-    // 3. Assign back to GraphNodes
-    for (let i = 0; i < N; i++) {
-        eligibleNodes[i].authorityScore = authScores[i];
-        eligibleNodes[i].hubScore = hubScores[i];
-    }
-    // 4. Classification Logic
-    classifyLinkRoles(eligibleNodes);
-}
-function classifyLinkRoles(nodes) {
-    if (nodes.length === 0)
-        return;
-    const authScores = nodes.map(n => n.authorityScore || 0).sort((a, b) => a - b);
-    const hubScores = nodes.map(n => n.hubScore || 0).sort((a, b) => a - b);
-    // Use 75th percentile as "high" threshold
-    // Using median (50th percentile) as per original implementation,
-    // but the comment said "Use 75th percentile" while code used median.
-    // I'll stick to median to avoid breaking existing behavior, but correct the comment or logic?
-    // The original code:
-    // const medianAuth = authScores[Math.floor(authScores.length / 2)];
-    // const isHighAuth = auth > medianAuth && auth > 0.0001;
-    // So it uses median. I'll keep it as median.
-    const medianAuth = authScores[Math.floor(authScores.length / 2)];
-    const medianHub = hubScores[Math.floor(hubScores.length / 2)];
-    const maxAuth = authScores[authScores.length - 1];
-    const maxHub = hubScores[hubScores.length - 1];
-    for (const node of nodes) {
-        const auth = node.authorityScore || 0;
-        const hub = node.hubScore || 0;
-        // A node is high if it's above median, OR if it's the max (to handle uniform distributions)
-        // auth > 0 check is essential.
-        const isHighAuth = (auth > medianAuth || (auth === maxAuth && auth > 0)) && auth > 0.00001;
-        const isHighHub = (hub > medianHub || (hub === maxHub && hub > 0)) && hub > 0.00001;
-        if (isHighAuth && isHighHub) {
-            node.linkRole = 'power';
-        }
-        else if (isHighAuth) {
-            node.linkRole = 'authority';
-        }
-        else if (isHighHub) {
-            node.linkRole = 'hub';
-        }
-        else if (auth > 0.00001 && hub > 0.00001) {
-            node.linkRole = 'balanced';
-        }
-        else {
-            node.linkRole = 'peripheral';
-        }
-    }
-}

package/scripts/copy-assets.js

@@ -1,37 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-// Ensure dist directories exist
-const reportDestDir = path.join(__dirname, '../dist/report');
-if (!fs.existsSync(reportDestDir)) {
-  fs.mkdirSync(reportDestDir, { recursive: true });
-}
-
-const analysisDestDir = path.join(__dirname, '../dist/analysis');
-if (!fs.existsSync(analysisDestDir)) {
-  fs.mkdirSync(analysisDestDir, { recursive: true });
-}
-
-// Copy Report Assets
-const crawlSrc = path.join(__dirname, '../src/report/crawl.html');
-const crawlDest = path.join(reportDestDir, 'crawl.html');
-if (fs.existsSync(crawlSrc)) {
-  fs.copyFileSync(crawlSrc, crawlDest);
-}
-
-// Copy Analysis Assets
-const analysisListSrc = path.join(__dirname, '../src/analysis/analysis_list.html');
-const analysisListDest = path.join(analysisDestDir, 'analysis_list.html');
-if (fs.existsSync(analysisListSrc)) {
-  fs.copyFileSync(analysisListSrc, analysisListDest);
-}
-
-const analysisPageSrc = path.join(__dirname, '../src/analysis/analysis_page.html');
-const analysisPageDest = path.join(analysisDestDir, 'analysis_page.html');
-if (fs.existsSync(analysisPageSrc)) {
-  fs.copyFileSync(analysisPageSrc, analysisPageDest);
-}

package/src/analysis/analysis_list.html

@@ -1,35 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8" />
-  <title>Crawlith Analysis Report</title>
-  <style>
-    body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; max-width: 1000px; margin: 0 auto; padding: 20px; color: #333; }
-    h1 { border-bottom: 2px solid #eee; padding-bottom: 10px; }
-    table { width: 100%; border-collapse: collapse; margin-top: 20px; }
-    th, td { padding: 8px 12px; border: 1px solid #ddd; text-align: left; }
-    th { background-color: #f4f4f4; }
-    tr:nth-child(even) { background-color: #f9f9f9; }
-    tr:hover { background-color: #f1f1f1; }
-  </style>
-</head>
-<body>
-  <h1>Analysis</h1>
-  <p>Pages: {{PAGES_ANALYZED}}</p>
-  <p>Average SEO: {{AVG_SEO_SCORE}}</p>
-  <table border="1" cellspacing="0" cellpadding="6">
-    <thead>
-      <tr>
-        <th>URL</th>
-        <th>SEO Score</th>
-        <th>Thin Score</th>
-        <th>Title</th>
-        <th>Meta</th>
-      </tr>
-    </thead>
-    <tbody>
-      {{ROWS}}
-    </tbody>
-  </table>
-</body>
-</html>

package/src/analysis/analysis_page.html

@@ -1,123 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Analysis for {{URL}}</title>
-    <style>
-        body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; line-height: 1.6; color: #333; }
-        h1 { border-bottom: 2px solid #eee; padding-bottom: 10px; }
-        h2 { margin-top: 30px; border-bottom: 1px solid #eee; padding-bottom: 5px; }
-        .score-card { display: flex; gap: 20px; margin-bottom: 30px; }
-        .score-box { background: #f8f9fa; padding: 15px; border-radius: 8px; text-align: center; flex: 1; border: 1px solid #e1e4e8; }
-        .score-val { font-size: 24px; font-weight: bold; color: #0366d6; }
-        .status-ok { color: green; font-weight: bold; }
-        .status-warning { color: orange; font-weight: bold; }
-        .status-critical { color: red; font-weight: bold; }
-        .status-missing { color: red; font-weight: bold; }
-        .data-table { width: 100%; border-collapse: collapse; margin-top: 10px; }
-        .data-table th, .data-table td { text-align: left; padding: 8px; border-bottom: 1px solid #eee; }
-        .data-table th { width: 150px; color: #666; }
-        code { background: #f6f8fa; padding: 2px 4px; border-radius: 3px; font-size: 0.9em; }
-    </style>
-  </head>
-  <body>
-    <h1>Page Analysis</h1>
-    <p><strong>URL:</strong> <a href="{{URL}}" target="_blank">{{URL}}</a></p>
-
-    <div class="score-card">
-      <div class="score-box">
-        <div class="score-val">{{SEO_SCORE}}</div>
-        <div>SEO Score</div>
-      </div>
-      <div class="score-box">
-        <div class="score-val">{{THIN_SCORE}}</div>
-        <div>Thin Content Score</div>
-      </div>
-      <div class="score-box">
-        <div class="score-val">{{HTTP_STATUS}}</div>
-        <div>HTTP Status</div>
-      </div>
-    </div>
-
-    <h2>Meta Tags</h2>
-    <table class="data-table">
-      <tr>
-        <th>Title</th>
-        <td>
-          <div>{{TITLE_VALUE}}</div>
-          <small>Length: {{TITLE_LENGTH}} | Status: <span class="status-{{TITLE_STATUS}}">{{TITLE_STATUS}}</span></small>
-        </td>
-      </tr>
-      <tr>
-        <th>Description</th>
-        <td>
-          <div>{{META_DESCRIPTION_VALUE}}</div>
-          <small>Length: {{META_DESCRIPTION_LENGTH}} | Status: <span class="status-{{META_DESCRIPTION_STATUS}}">{{META_DESCRIPTION_STATUS}}</span></small>
-        </td>
-      </tr>
-      <tr>
-        <th>Canonical</th>
-        <td>{{CANONICAL}}</td>
-      </tr>
-      <tr>
-        <th>Robots</th>
-        <td>
-          Index: {{ROBOTS_INDEX}},
-          Follow: {{ROBOTS_FOLLOW}}
-        </td>
-      </tr>
-    </table>
-
-    <h2>Content & Heading</h2>
-    <table class="data-table">
-      <tr>
-        <th>H1 Tag</th>
-        <td>
-          Status: <span class="status-{{H1_STATUS}}">{{H1_STATUS}}</span>
-          ({{H1_COUNT}} detected)
-          {{H1_MATCHES_TITLE}}
-        </td>
-      </tr>
-      <tr>
-        <th>Word Count</th>
-        <td>{{WORD_COUNT}} words</td>
-      </tr>
-      <tr>
-        <th>Unique Sentences</th>
-        <td>{{UNIQUE_SENTENCES}}</td>
-      </tr>
-      <tr>
-        <th>Text / HTML Ratio</th>
-        <td>{{TEXT_HTML_RATIO}}%</td>
-      </tr>
-    </table>
-
-    <h2>Links & Images</h2>
-    <table class="data-table">
-      <tr>
-        <th>Internal Links</th>
-        <td>{{INTERNAL_LINKS}}</td>
-      </tr>
-      <tr>
-        <th>External Links</th>
-        <td>{{EXTERNAL_LINKS}} ({{EXTERNAL_RATIO}}%)</td>
-      </tr>
-      <tr>
-        <th>Images</th>
-        <td>{{TOTAL_IMAGES}} total ({{MISSING_ALT}} missing alt text)</td>
-      </tr>
-    </table>
-
-    <h2>Structured Data</h2>
-    <table class="data-table">
-      <tr>
-        <th>Status</th>
-        <td>
-          {{STRUCTURED_DATA_STATUS}}
-        </td>
-      </tr>
-      {{STRUCTURED_DATA_TYPES_ROW}}
-    </table>
-  </body>
-</html>