@crawlith/core 0.1.0 → 0.1.1

package/dist/events.d.ts

@@ -0,0 +1,48 @@
+export type CrawlEvent = {
+    type: 'crawl:start';
+    url: string;
+} | {
+    type: 'crawl:success';
+    url: string;
+    status: number;
+    durationMs: number;
+    depth?: number;
+} | {
+    type: 'crawl:error';
+    url: string;
+    error: string;
+    depth?: number;
+} | {
+    type: 'crawl:limit-reached';
+    limit: number;
+} | {
+    type: 'queue:enqueue';
+    url: string;
+    depth: number;
+} | {
+    type: 'metrics:start';
+    phase: string;
+} | {
+    type: 'metrics:complete';
+    durationMs: number;
+} | {
+    type: 'debug';
+    message: string;
+    context?: unknown;
+} | {
+    type: 'info';
+    message: string;
+    context?: unknown;
+} | {
+    type: 'warn';
+    message: string;
+    context?: unknown;
+} | {
+    type: 'error';
+    message: string;
+    error?: unknown;
+    context?: unknown;
+};
+export interface EngineContext {
+    emit: (event: CrawlEvent) => void;
+}

package/dist/events.js

@@ -0,0 +1 @@
+export {};

package/dist/graph/cluster.js

@@ -1,4 +1,5 @@
 import { SimHash } from './simhash.js';
+import { load } from 'cheerio';
 /**
  * Detects content clusters using 64-bit SimHash and Hamming Distance.
  * Uses band optimization to reduce O(n^2) comparisons.
@@ -11,21 +12,19 @@ export function detectContentClusters(graph, threshold = 10, minSize = 3) {
     // Banding Optimization (4 bands of 16 bits)
     // Note: For threshold > 3, this is a heuristic and may miss some pairs,
     // but it dramatically reduces the search space as requested.
-    const bands = 4;
-    const bandWidth = 16;
-    const buckets = Array.from({ length: bands }, () => new Map());
+    const buckets = Array.from({ length: SimHash.BANDS }, () => new Map());
     for (const node of nodes) {
         const hash = BigInt(node.simhash);
-        for (let b = 0; b < bands; b++) {
-            const bandValue = Number((hash >> BigInt(b * bandWidth)) & 0xffffn);
+        const bandValues = SimHash.getBands(hash);
+        bandValues.forEach((bandValue, b) => {
             if (!buckets[b].has(bandValue)) {
                 buckets[b].set(bandValue, new Set());
             }
             buckets[b].get(bandValue).add(node.url);
-        }
+        });
     }
     const checkedPairs = new Set();
-    for (let b = 0; b < bands; b++) {
+    for (let b = 0; b < SimHash.BANDS; b++) {
         for (const bucket of buckets[b].values()) {
             if (bucket.size < 2)
                 continue;
@@ -135,15 +134,64 @@ function selectPrimaryUrl(urls, graph) {
  * Calculates cannibalization risk based on title and H1 similarity within the cluster.
  */
 function calculateClusterRisk(nodes) {
-    // Logic: Check if there's significant overlap in Titles or H1s among cluster members.
-    // This is a heuristic as requested.
-    // Simplified heuristic: risk is based on cluster density and size
-    // Large clusters of highly similar content are high risk.
-    // Fallback to a safe categorization
-    if (nodes.length > 5)
+    if (nodes.length <= 1)
+        return 'low';
+    // Count title and H1 occurrences
+    const titleCounts = new Map();
+    const h1Counts = new Map();
+    let processedCount = 0;
+    for (const node of nodes) {
+        if (!node.html)
+            continue;
+        try {
+            const $ = load(node.html);
+            const title = $('title').text().trim().toLowerCase();
+            const h1 = $('h1').first().text().trim().toLowerCase();
+            if (title) {
+                titleCounts.set(title, (titleCounts.get(title) || 0) + 1);
+            }
+            if (h1) {
+                h1Counts.set(h1, (h1Counts.get(h1) || 0) + 1);
+            }
+            processedCount++;
+        }
+        catch {
+            // Ignore parsing errors
+        }
+    }
+    // If we couldn't parse enough content (e.g., no HTML stored), fallback to size-based heuristic
+    if (processedCount < nodes.length * 0.5) {
+        if (nodes.length > 5)
+            return 'high';
+        if (nodes.length > 2)
+            return 'medium';
+        return 'low';
+    }
+    // Calculate duplicate ratios
+    let duplicateTitleCount = 0;
+    let duplicateH1Count = 0;
+    for (const count of titleCounts.values()) {
+        if (count > 1)
+            duplicateTitleCount += count;
+    }
+    for (const count of h1Counts.values()) {
+        if (count > 1)
+            duplicateH1Count += count;
+    }
+    const titleDupeRatio = duplicateTitleCount / nodes.length;
+    const h1DupeRatio = duplicateH1Count / nodes.length;
+    // Heuristic 1: High Risk
+    // Significant overlap in Titles OR H1s (e.g., > 30% of cluster members are duplicates)
+    if (titleDupeRatio > 0.3 || h1DupeRatio > 0.3) {
         return 'high';
-    if (nodes.length > 2)
+    }
+    // Heuristic 2: Medium Risk
+    // Any overlap, or very large clusters (potential template issues or thin content)
+    if (titleDupeRatio > 0 || h1DupeRatio > 0 || nodes.length > 10) {
         return 'medium';
+    }
+    // Heuristic 3: Low Risk
+    // Unique content and manageable cluster size
     return 'low';
 }
 /**

package/dist/graph/duplicate.js

@@ -1,3 +1,4 @@
+import { Graph } from './graph.js';
 import { SimHash } from './simhash.js';
 /**
  * Detects exact and near duplicates, identifies canonical conflicts,
@@ -6,15 +7,30 @@ import { SimHash } from './simhash.js';
 export function detectDuplicates(graph, options = {}) {
     const collapse = options.collapse !== false; // Default to true
     const threshold = options.simhashThreshold ?? 3;
-    const exactClusters = [];
-    const nearClusters = [];
     const nodes = graph.getNodes();
+    let clusterCounter = 1;
     // Phase 1 & 2: Exact Duplicate Detection
+    const { exactClusters, nearCandidates, nextId: nextId1 } = findExactDuplicates(nodes, clusterCounter);
+    clusterCounter = nextId1;
+    // Phase 3: Near Duplicate Detection
+    const { nearClusters } = findNearDuplicates(nearCandidates, threshold, clusterCounter);
+    const allClusters = [...exactClusters, ...nearClusters];
+    // Phase 4, 5, 6: Process Clusters (Template-Heavy, Canonical, Representative)
+    processClusters(allClusters, graph, collapse);
+    // Final Edge Transfer if Collapsing
+    if (collapse) {
+        collapseEdges(graph);
+    }
+}
+function findExactDuplicates(nodes, startId) {
+    const exactMap = groupNodesByContentHash(nodes);
+    return createExactClusters(exactMap, startId);
+}
+function groupNodesByContentHash(nodes) {
     const exactMap = new Map();
     for (const node of nodes) {
         if (!node.contentHash || node.status !== 200)
             continue;
-        // Safety check: if there's no soft404 signal (soft404 is handled elsewhere, but just filter 200 OKs)
         let arr = exactMap.get(node.contentHash);
         if (!arr) {
             arr = [];
@@ -22,14 +38,16 @@ export function detectDuplicates(graph, options = {}) {
         }
         arr.push(node);
     }
-    // Nodes that are NOT part of an exact duplicate group are candidates for near duplicate checks
+    return exactMap;
+}
+function createExactClusters(exactMap, startId) {
+    const exactClusters = [];
     const nearCandidates = [];
-    let clusterCounter = 1;
+    let clusterCounter = startId;
     for (const [_hash, group] of exactMap.entries()) {
         if (group.length > 1) {
             const id = `cluster_exact_${clusterCounter++}`;
             exactClusters.push({ id, type: 'exact', nodes: group });
-            // Mark nodes
             for (const n of group) {
                 n.duplicateClusterId = id;
                 n.duplicateType = 'exact';
@@ -39,213 +57,246 @@ export function detectDuplicates(graph, options = {}) {
             nearCandidates.push(group[0]);
         }
     }
-    // Phase 3: Near Duplicate Detection (SimHash with Bands)
-    // 64-bit simhash -> split into 4 bands of 16 bits.
-    const bandsMaps = [
-        new Map(),
-        new Map(),
-        new Map(),
-        new Map()
-    ];
-    for (const node of nearCandidates) {
-        if (!node.simhash)
-            continue;
-        const simhash = BigInt(node.simhash);
-        // Extract 16 bit bands
-        const b0 = Number(simhash & 0xffffn);
-        const b1 = Number((simhash >> 16n) & 0xffffn);
-        const b2 = Number((simhash >> 32n) & 0xffffn);
-        const b3 = Number((simhash >> 48n) & 0xffffn);
-        const bands = [b0, b1, b2, b3];
-        for (let i = 0; i < 4; i++) {
-            let arr = bandsMaps[i].get(bands[i]);
+    return { exactClusters, nearCandidates, nextId: clusterCounter };
+}
+function findNearDuplicates(candidates, threshold, startId) {
+    const { bandsMaps, simhashes } = buildSimHashBuckets(candidates);
+    const { parent, involvedIndices } = findConnectedComponents(bandsMaps, simhashes, candidates.length, threshold);
+    return extractClusters(parent, involvedIndices, candidates, startId);
+}
+function buildSimHashBuckets(candidates) {
+    const n = candidates.length;
+    const simhashes = new BigUint64Array(n);
+    const validIndices = [];
+    for (let i = 0; i < n; i++) {
+        if (candidates[i].simhash) {
+            simhashes[i] = BigInt(candidates[i].simhash);
+            validIndices.push(i);
+        }
+    }
+    const bandsMaps = Array.from({ length: SimHash.BANDS }, () => new Map());
+    for (const idx of validIndices) {
+        const bands = SimHash.getBands(simhashes[idx]);
+        for (let b = 0; b < SimHash.BANDS; b++) {
+            let arr = bandsMaps[b].get(bands[b]);
             if (!arr) {
                 arr = [];
-                bandsMaps[i].set(bands[i], arr);
+                bandsMaps[b].set(bands[b], arr);
             }
-            arr.push(node);
+            arr.push(idx);
         }
     }
-    // Find candidate pairs
-    const nearGroupMap = new Map(); // node.url -> cluster set
-    const checkedPairs = new Set();
-    for (let i = 0; i < 4; i++) {
-        for (const [_bandVal, bucketNodes] of bandsMaps[i].entries()) {
-            if (bucketNodes.length < 2)
-                continue; // nothing to compare
-            // Compare all nodes in this bucket
-            for (let j = 0; j < bucketNodes.length; j++) {
-                for (let k = j + 1; k < bucketNodes.length; k++) {
-                    const n1 = bucketNodes[j];
-                    const n2 = bucketNodes[k];
-                    // Ensure n1 < n2 lexicographically to avoid duplicate pairs
-                    const [a, b] = n1.url < n2.url ? [n1, n2] : [n2, n1];
-                    const pairKey = `${a.url}|${b.url}`;
-                    if (checkedPairs.has(pairKey))
-                        continue;
-                    checkedPairs.add(pairKey);
-                    const dist = SimHash.hammingDistance(BigInt(a.simhash), BigInt(b.simhash));
+    return { bandsMaps, simhashes, validIndices };
+}
+function findConnectedComponents(bandsMaps, simhashes, n, threshold) {
+    // Union-Find Arrays (Integer-based)
+    const parent = new Uint32Array(n);
+    const rank = new Uint8Array(n);
+    for (let i = 0; i < n; i++) {
+        parent[i] = i;
+        rank[i] = 0;
+    }
+    function find(i) {
+        let root = i;
+        while (parent[root] !== root) {
+            root = parent[root];
+        }
+        let curr = i;
+        while (curr !== root) {
+            const next = parent[curr];
+            parent[curr] = root;
+            curr = next;
+        }
+        return root;
+    }
+    function union(i, j) {
+        const rootI = find(i);
+        const rootJ = find(j);
+        if (rootI !== rootJ) {
+            const rankI = rank[rootI];
+            const rankJ = rank[rootJ];
+            if (rankI < rankJ) {
+                parent[rootI] = rootJ;
+            }
+            else if (rankI > rankJ) {
+                parent[rootJ] = rootI;
+            }
+            else {
+                parent[rootJ] = rootI;
+                rank[rootI]++;
+            }
+        }
+    }
+    const involvedIndices = new Set();
+    for (let b = 0; b < SimHash.BANDS; b++) {
+        for (const bucketIndices of bandsMaps[b].values()) {
+            if (bucketIndices.length < 2)
+                continue;
+            for (let j = 0; j < bucketIndices.length; j++) {
+                for (let k = j + 1; k < bucketIndices.length; k++) {
+                    const idx1 = bucketIndices[j];
+                    const idx2 = bucketIndices[k];
+                    const root1 = find(idx1);
+                    const root2 = find(idx2);
+                    if (root1 === root2)
+                        continue; // Already connected, skip expensive distance check
+                    const dist = SimHash.hammingDistance(simhashes[idx1], simhashes[idx2]);
                     if (dist <= threshold) {
-                        // They are near duplicates. 
-                        // Find or create their cluster set using union-find or reference propagation
-                        const setA = nearGroupMap.get(a.url);
-                        const setB = nearGroupMap.get(b.url);
-                        if (!setA && !setB) {
-                            const newSet = new Set([a, b]);
-                            nearGroupMap.set(a.url, newSet);
-                            nearGroupMap.set(b.url, newSet);
-                        }
-                        else if (setA && !setB) {
-                            setA.add(b);
-                            nearGroupMap.set(b.url, setA);
-                        }
-                        else if (setB && !setA) {
-                            setB.add(a);
-                            nearGroupMap.set(a.url, setB);
-                        }
-                        else if (setA && setB && setA !== setB) {
-                            // Merge sets
-                            for (const node of setB) {
-                                setA.add(node);
-                                nearGroupMap.set(node.url, setA);
-                            }
-                        }
+                        union(root1, root2);
+                        involvedIndices.add(idx1);
+                        involvedIndices.add(idx2);
                     }
                 }
             }
         }
     }
-    // Compile near duplicate clusters (deduplicated by Set reference)
-    const uniqueNearSets = new Set();
-    for (const group of nearGroupMap.values()) {
-        uniqueNearSets.add(group);
+    return { parent, involvedIndices };
+}
+function extractClusters(parent, involvedIndices, candidates, startId) {
+    const nearClusters = [];
+    let clusterCounter = startId;
+    function find(i) {
+        let root = i;
+        while (parent[root] !== root) {
+            root = parent[root];
+        }
+        let curr = i;
+        while (curr !== root) {
+            const next = parent[curr];
+            parent[curr] = root;
+            curr = next;
+        }
+        return root;
+    }
+    // Compile clusters
+    const clusterMap = new Map();
+    for (const idx of involvedIndices) {
+        const root = find(idx);
+        let group = clusterMap.get(root);
+        if (!group) {
+            group = [];
+            clusterMap.set(root, group);
+        }
+        group.push(idx);
     }
-    for (const groupSet of uniqueNearSets) {
-        if (groupSet.size > 1) {
+    for (const groupIndices of clusterMap.values()) {
+        if (groupIndices.length > 1) {
             const id = `cluster_near_${clusterCounter++}`;
-            const groupArr = Array.from(groupSet);
-            nearClusters.push({ id, type: 'near', nodes: groupArr });
-            for (const n of groupArr) {
+            const groupNodes = groupIndices.map(idx => candidates[idx]);
+            nearClusters.push({ id, type: 'near', nodes: groupNodes });
+            for (const n of groupNodes) {
                 n.duplicateClusterId = id;
                 n.duplicateType = 'near';
             }
         }
     }
-    const allClusters = [...exactClusters, ...nearClusters];
-    // Phase 4: Template-Heavy Detection
-    // Mark classes as 'template_heavy' if ratio < 0.3
-    for (const cluster of allClusters) {
-        const avgRatio = cluster.nodes.reduce((sum, n) => sum + (n.uniqueTokenRatio || 0), 0) / cluster.nodes.length;
-        if (avgRatio < 0.3) {
-            cluster.type = 'template_heavy';
-            cluster.nodes.forEach(n => n.duplicateType = 'template_heavy');
-        }
+    return { nearClusters, nextId: clusterCounter };
+}
+function processClusters(clusters, graph, collapse) {
+    for (const cluster of clusters) {
+        processSingleCluster(cluster, graph, collapse);
+    }
+}
+function processSingleCluster(cluster, graph, collapse) {
+    checkTemplateHeavy(cluster);
+    cluster.severity = calculateSeverity(cluster);
+    const representative = selectRepresentative(cluster);
+    cluster.representative = representative.url;
+    applyClusterToGraph(cluster, representative, graph, collapse);
+}
+function checkTemplateHeavy(cluster) {
+    const avgRatio = cluster.nodes.reduce((sum, n) => sum + (n.uniqueTokenRatio || 0), 0) / cluster.nodes.length;
+    if (avgRatio < 0.3) {
+        cluster.type = 'template_heavy';
+        cluster.nodes.forEach(n => n.duplicateType = 'template_heavy');
+    }
+}
+function calculateSeverity(cluster) {
+    const canonicals = new Set();
+    let hasMissing = false;
+    for (const n of cluster.nodes) {
+        if (!n.canonical)
+            hasMissing = true;
+        else
+            canonicals.add(n.canonical);
+    }
+    if (hasMissing || canonicals.size > 1) {
+        return 'high';
+    }
+    else if (cluster.type === 'near') {
+        return 'medium';
     }
-    // Phase 5: Canonical Conflict & Representative Selection
-    for (const cluster of allClusters) {
-        const canonicals = new Set();
-        let hasMissing = false;
+    else {
+        return 'low';
+    }
+}
+function selectRepresentative(cluster) {
+    const urlsInCluster = new Set(cluster.nodes.map(n => n.url));
+    const validCanonicals = cluster.nodes.filter(n => n.canonical && urlsInCluster.has(n.canonical) && n.url === n.canonical);
+    if (validCanonicals.length > 0) {
+        return validCanonicals[0];
+    }
+    return cluster.nodes.reduce((best, current) => {
+        if (current.inLinks > best.inLinks)
+            return current;
+        if (current.inLinks < best.inLinks)
+            return best;
+        if (current.url.length < best.url.length)
+            return current;
+        return best;
+    });
+}
+function applyClusterToGraph(cluster, representative, graph, collapse) {
+    cluster.nodes.forEach(n => {
+        n.isClusterPrimary = n.url === representative.url;
+        n.isCollapsed = false;
+        n.collapseInto = undefined;
+    });
+    graph.duplicateClusters.push({
+        id: cluster.id,
+        type: cluster.type,
+        size: cluster.nodes.length,
+        representative: representative.url,
+        severity: cluster.severity
+    });
+    if (collapse) {
         for (const n of cluster.nodes) {
-            if (!n.canonical)
-                hasMissing = true;
-            // We compare full absolute canonical URLs (assuming they are normalized during crawl)
-            else
-                canonicals.add(n.canonical);
-        }
-        if (hasMissing || canonicals.size > 1) {
-            cluster.severity = 'high';
-        }
-        else if (cluster.type === 'near') {
-            cluster.severity = 'medium';
-        }
-        else {
-            cluster.severity = 'low';
-        }
-        // Phase 6: Select Representative
-        // 1. Valid Canonical target in cluster
-        // 2. Highest internal in-degree
-        // 3. Shortest URL
-        // 4. First discovered (relying on array order, which is from BFS map roughly)
-        let representativeNode = cluster.nodes[0];
-        // Evaluate best rep
-        const urlsInCluster = new Set(cluster.nodes.map(n => n.url));
-        const validCanonicals = cluster.nodes.filter(n => n.canonical && urlsInCluster.has(n.canonical) && n.url === n.canonical);
-        if (validCanonicals.length > 0) {
-            representativeNode = validCanonicals[0]; // If multiple, just pick first matching self
-        }
-        else {
-            representativeNode = cluster.nodes.reduce((best, current) => {
-                if (current.inLinks > best.inLinks)
-                    return current;
-                if (current.inLinks < best.inLinks)
-                    return best;
-                if (current.url.length < best.url.length)
-                    return current;
-                return best;
-            });
-        }
-        cluster.representative = representativeNode.url;
-        cluster.nodes.forEach(n => {
-            n.isClusterPrimary = n.url === representativeNode.url;
-            n.isCollapsed = false; // default for JSON
-            n.collapseInto = undefined;
-        });
-        // Push to Graph's final cluster list
-        graph.duplicateClusters.push({
-            id: cluster.id,
-            type: cluster.type,
-            size: cluster.nodes.length,
-            representative: representativeNode.url,
-            severity: cluster.severity
-        });
-        // Controlled Collapse
-        if (collapse) {
-            for (const n of cluster.nodes) {
-                if (n.url !== representativeNode.url) {
-                    n.isCollapsed = true;
-                    n.collapseInto = representativeNode.url;
-                }
+            if (n.url !== representative.url) {
+                n.isCollapsed = true;
+                n.collapseInto = representative.url;
             }
         }
     }
-    // Final Edge Transfer if Collapsing
-    if (collapse) {
-        const edges = graph.getEdges();
-        const updatedEdges = new Map();
-        for (const edge of edges) {
-            const sourceNode = graph.nodes.get(edge.source);
-            const targetNode = graph.nodes.get(edge.target);
-            if (!sourceNode || !targetNode)
-                continue;
-            // We do NOT modify source structure for out-bound edges of collapsed nodes? 
-            // Spec: "Ignore edges from collapsed nodes. Transfer inbound edges to representative."
-            // Actually, if a node links TO a collapsed node, we repoint the edge to the representative.
-            // If a collapsed node links to X, we ignore it (PageRank will filter it out).
-            const actualSource = edge.source;
-            // repoint target
-            const actualTarget = targetNode.isCollapsed && targetNode.collapseInto ? targetNode.collapseInto : edge.target;
-            // Skip self-referential edges caused by repointing
-            if (actualSource === actualTarget)
-                continue;
-            const edgeKey = `${actualSource}|${actualTarget}`;
-            const existingWeight = updatedEdges.get(edgeKey) || 0;
-            updatedEdges.set(edgeKey, Math.max(existingWeight, edge.weight)); // deduplicate
-        }
-        // Update graph edges in-place
-        graph.edges = updatedEdges;
-        // Re-calculate inLinks and outLinks based on collapsed edges
-        for (const node of graph.getNodes()) {
-            node.inLinks = 0;
-            node.outLinks = 0;
-        }
-        for (const [edgeKey, _weight] of updatedEdges.entries()) {
-            const [src, tgt] = edgeKey.split('|');
-            const sn = graph.nodes.get(src);
-            const tn = graph.nodes.get(tgt);
-            if (sn)
-                sn.outLinks++;
-            if (tn)
-                tn.inLinks++;
-        }
+}
+function collapseEdges(graph) {
+    const edges = graph.getEdges();
+    const updatedEdges = new Map();
+    for (const edge of edges) {
+        const sourceNode = graph.nodes.get(edge.source);
+        const targetNode = graph.nodes.get(edge.target);
+        if (!sourceNode || !targetNode)
+            continue;
+        const actualSource = edge.source;
+        const actualTarget = targetNode.isCollapsed && targetNode.collapseInto ? targetNode.collapseInto : edge.target;
+        if (actualSource === actualTarget)
+            continue;
+        const edgeKey = Graph.getEdgeKey(actualSource, actualTarget);
+        const existingWeight = updatedEdges.get(edgeKey) || 0;
+        updatedEdges.set(edgeKey, Math.max(existingWeight, edge.weight));
+    }
+    graph.edges = updatedEdges;
+    // Re-calculate inLinks and outLinks based on collapsed edges
+    for (const node of graph.getNodes()) {
+        node.inLinks = 0;
+        node.outLinks = 0;
+    }
+    for (const [edgeKey, _weight] of updatedEdges.entries()) {
+        const { source: src, target: tgt } = Graph.parseEdgeKey(edgeKey);
+        const sn = graph.nodes.get(src);
+        const tn = graph.nodes.get(tgt);
+        if (sn)
+            sn.outLinks++;
+        if (tn)
+            tn.inLinks++;
     }
 }