@crawlith/core 0.1.1 → 0.1.2

package/dist/analysis/clustering.d.ts

@@ -0,0 +1,23 @@
+import { Graph } from '../graph/graph.js';
+export interface ClusterInfo {
+    id: number;
+    count: number;
+    primaryUrl: string;
+    risk: 'low' | 'medium' | 'high';
+    sharedPathPrefix?: string;
+    nodes?: string[];
+}
+export interface ClusteringOptions {
+    threshold?: number;
+    minSize?: number;
+}
+export declare class ClusteringService {
+    /**
+     * Detects content clusters using 64-bit SimHash and Hamming Distance.
+     * Uses band optimization to reduce O(n^2) comparisons.
+     */
+    detectContentClusters(graph: Graph, threshold?: number, minSize?: number): ClusterInfo[];
+    private selectPrimaryUrl;
+    private calculateClusterRisk;
+    private findSharedPathPrefix;
+}

package/dist/analysis/clustering.js

@@ -0,0 +1,206 @@
+import { SimHash } from '../graph/simhash.js';
+import { analyzeH1, analyzeTitle } from './seo.js';
+export class ClusteringService {
+    /**
+     * Detects content clusters using 64-bit SimHash and Hamming Distance.
+     * Uses band optimization to reduce O(n^2) comparisons.
+     */
+    detectContentClusters(graph, threshold = 10, minSize = 3) {
+        const nodes = graph.getNodes().filter(n => n.simhash && n.status === 200);
+        if (nodes.length === 0)
+            return [];
+        const adjacency = new Map();
+        // Banding Optimization (4 bands of 16 bits)
+        const buckets = Array.from({ length: SimHash.BANDS }, () => new Map());
+        for (const node of nodes) {
+            const hash = BigInt(node.simhash);
+            const bandValues = SimHash.getBands(hash);
+            bandValues.forEach((bandValue, b) => {
+                if (!buckets[b].has(bandValue)) {
+                    buckets[b].set(bandValue, []);
+                }
+                buckets[b].get(bandValue).push(node.url);
+            });
+        }
+        const checkedPairs = new Set();
+        for (let b = 0; b < SimHash.BANDS; b++) {
+            for (const bucket of buckets[b].values()) {
+                if (bucket.length < 2)
+                    continue;
+                const bucketNodes = bucket;
+                for (let i = 0; i < bucketNodes.length; i++) {
+                    for (let j = i + 1; j < bucketNodes.length; j++) {
+                        const u1 = bucketNodes[i];
+                        const u2 = bucketNodes[j];
+                        if (u1 === u2)
+                            continue;
+                        const pairKey = u1 < u2 ? `${u1}|${u2}` : `${u2}|${u1}`;
+                        if (checkedPairs.has(pairKey))
+                            continue;
+                        checkedPairs.add(pairKey);
+                        const n1 = graph.nodes.get(u1);
+                        const n2 = graph.nodes.get(u2);
+                        const dist = SimHash.hammingDistance(BigInt(n1.simhash), BigInt(n2.simhash));
+                        if (dist <= threshold) {
+                            if (!adjacency.has(u1))
+                                adjacency.set(u1, new Set());
+                            if (!adjacency.has(u2))
+                                adjacency.set(u2, new Set());
+                            adjacency.get(u1).add(u2);
+                            adjacency.get(u2).add(u1);
+                        }
+                    }
+                }
+            }
+        }
+        // Find connected components (Clusters)
+        const visited = new Set();
+        const clusters = [];
+        for (const node of nodes) {
+            if (visited.has(node.url))
+                continue;
+            const component = [];
+            const queue = [node.url];
+            visited.add(node.url);
+            while (queue.length > 0) {
+                const current = queue.shift();
+                component.push(current);
+                const neighbors = adjacency.get(current);
+                if (neighbors) {
+                    for (const neighbor of neighbors) {
+                        if (!visited.has(neighbor)) {
+                            visited.add(neighbor);
+                            queue.push(neighbor);
+                        }
+                    }
+                }
+            }
+            if (component.length >= minSize) {
+                clusters.push(component);
+            }
+        }
+        // Sort clusters by size (descending) then by primary URL (ascending) for deterministic IDs
+        clusters.sort((a, b) => {
+            if (b.length !== a.length)
+                return b.length - a.length;
+            const aPrimary = this.selectPrimaryUrl(a, graph);
+            const bPrimary = this.selectPrimaryUrl(b, graph);
+            return aPrimary.localeCompare(bPrimary);
+        });
+        const clusterInfos = [];
+        clusters.forEach((memberUrls, index) => {
+            const clusterId = index + 1;
+            const clusterNodes = memberUrls.map(url => graph.nodes.get(url));
+            for (const node of clusterNodes) {
+                node.clusterId = clusterId;
+            }
+            const primaryUrl = this.selectPrimaryUrl(memberUrls, graph);
+            const risk = this.calculateClusterRisk(clusterNodes);
+            const sharedPathPrefix = this.findSharedPathPrefix(memberUrls);
+            clusterInfos.push({
+                id: clusterId,
+                count: memberUrls.length,
+                primaryUrl,
+                risk,
+                sharedPathPrefix,
+                nodes: memberUrls
+            });
+        });
+        return clusterInfos;
+    }
+    selectPrimaryUrl(urls, graph) {
+        return urls.reduce((best, current) => {
+            const nBest = graph.nodes.get(best);
+            const nCurrent = graph.nodes.get(current);
+            if (nCurrent.inLinks > nBest.inLinks)
+                return current;
+            if (nCurrent.inLinks < nBest.inLinks)
+                return best;
+            if (current.length < best.length)
+                return current;
+            if (current.length > best.length)
+                return best;
+            return current.localeCompare(best) < 0 ? current : best;
+        });
+    }
+    calculateClusterRisk(nodes) {
+        if (nodes.length <= 1)
+            return 'low';
+        const titleCounts = new Map();
+        const h1Counts = new Map();
+        let processedCount = 0;
+        for (const node of nodes) {
+            if (!node.html)
+                continue;
+            const titleRes = analyzeTitle(node.html);
+            const h1Res = analyzeH1(node.html, titleRes.value);
+            const title = titleRes.value || '';
+            const h1 = h1Res.value || '';
+            if (title) {
+                titleCounts.set(title.toLowerCase(), (titleCounts.get(title.toLowerCase()) || 0) + 1);
+            }
+            if (h1) {
+                h1Counts.set(h1.toLowerCase(), (h1Counts.get(h1.toLowerCase()) || 0) + 1);
+            }
+            processedCount++;
+        }
+        if (processedCount < nodes.length * 0.5) {
+            if (nodes.length > 5)
+                return 'high';
+            if (nodes.length > 2)
+                return 'medium';
+            return 'low';
+        }
+        let duplicateTitleCount = 0;
+        let duplicateH1Count = 0;
+        for (const count of titleCounts.values()) {
+            if (count > 1)
+                duplicateTitleCount += count;
+        }
+        for (const count of h1Counts.values()) {
+            if (count > 1)
+                duplicateH1Count += count;
+        }
+        const titleDupeRatio = duplicateTitleCount / nodes.length;
+        const h1DupeRatio = duplicateH1Count / nodes.length;
+        if (titleDupeRatio > 0.3 || h1DupeRatio > 0.3) {
+            return 'high';
+        }
+        if (titleDupeRatio > 0 || h1DupeRatio > 0 || nodes.length > 10) {
+            return 'medium';
+        }
+        return 'low';
+    }
+    findSharedPathPrefix(urls) {
+        if (urls.length < 2)
+            return undefined;
+        try {
+            // Works for both absolute URLs (https://example.com/foo) and root-relative paths (/foo)
+            const getPathname = (u) => {
+                try {
+                    return new URL(u).pathname.split('/').filter(Boolean);
+                }
+                catch {
+                    // root-relative path like '/foo/bar'
+                    return u.split('/').filter(Boolean);
+                }
+            };
+            const paths = urls.map(getPathname);
+            const first = paths[0];
+            const common = [];
+            for (let i = 0; i < first.length; i++) {
+                const segment = first[i];
+                if (paths.every(p => p[i] === segment)) {
+                    common.push(segment);
+                }
+                else {
+                    break;
+                }
+            }
+            return common.length > 0 ? '/' + common.join('/') : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+}

package/dist/analysis/content.d.ts

@@ -8,5 +8,5 @@ export interface ThinScoreWeights {
     ratioWeight: number;
     dupWeight: number;
 }
-export declare function analyzeContent(html: string): ContentAnalysis;
+export declare function analyzeContent($: any): ContentAnalysis;
 export declare function calculateThinContentScore(content: ContentAnalysis, duplicationScore: number, weights?: ThinScoreWeights): number;

package/dist/analysis/content.js

@@ -4,14 +4,20 @@ const DEFAULT_WEIGHTS = {
     ratioWeight: 0.35,
     dupWeight: 0.25
 };
-export function analyzeContent(html) {
-    const $ = load(html || '<html></html>');
-    $('script,style,nav,footer').remove();
-    const text = $('body').length ? $('body').text() : $.text();
+export function analyzeContent($) {
+    const isString = typeof $ === 'string';
+    const cheerioObj = isString ? load($ || '<html></html>') : $;
+    // We don't want to modify the shared $ object if we remove elements
+    // So we create a localized copy of the body text or use selection
+    const body = cheerioObj('body').length ? cheerioObj('body') : cheerioObj('html');
+    // To avoid removing from shared $, we extract text from a clone if possible, 
+    // but cloning in cheerio is expensive. 
+    // Better: just get the text and clean it or use a filter.
+    const text = body.clone().find('script,style,nav,footer').remove().end().text();
     const cleanText = text.replace(/\s+/g, ' ').trim();
     const words = cleanText ? cleanText.split(/\s+/).filter(Boolean) : [];
     const wordCount = words.length;
-    const htmlLength = Math.max(html.length, 1);
+    const htmlLength = isString ? ($.length || 1) : 1000; // Fallback if we don't have original HTML length
     const textHtmlRatio = cleanText.length / htmlLength;
     const sentenceSet = new Set(cleanText
         .split(/[.!?]+/)

package/dist/analysis/duplicate.d.ts

@@ -0,0 +1,34 @@
+import { Graph, GraphNode } from '../graph/graph.js';
+export interface DuplicateOptions {
+    collapse?: boolean;
+    simhashThreshold?: number;
+}
+export interface DuplicateCluster {
+    id: string;
+    type: 'exact' | 'near' | 'template_heavy';
+    nodes: GraphNode[];
+    size: number;
+    representative?: string;
+    severity?: 'low' | 'medium' | 'high';
+}
+export declare class DuplicateService {
+    /**
+     * Detects exact and near duplicates, identifies canonical conflicts,
+     * and performs non-destructive collapse of edges.
+     */
+    detectDuplicates(graph: Graph, options?: DuplicateOptions): DuplicateCluster[];
+    private findExactDuplicates;
+    private groupNodesByContentHash;
+    private createExactClusters;
+    private findNearDuplicates;
+    private buildSimHashBuckets;
+    private findConnectedComponents;
+    private extractClusters;
+    private processClusters;
+    private processSingleCluster;
+    private checkTemplateHeavy;
+    private calculateSeverity;
+    private selectRepresentative;
+    private applyClusterToGraph;
+    private collapseEdges;
+}

package/dist/analysis/duplicate.js

@@ -0,0 +1,305 @@
+import { Graph } from '../graph/graph.js';
+import { SimHash } from '../graph/simhash.js';
+export class DuplicateService {
+    /**
+     * Detects exact and near duplicates, identifies canonical conflicts,
+     * and performs non-destructive collapse of edges.
+     */
+    detectDuplicates(graph, options = {}) {
+        const collapse = options.collapse !== false; // Default to true
+        const threshold = options.simhashThreshold ?? 3;
+        const nodes = graph.getNodes();
+        let clusterCounter = 1;
+        // Phase 1 & 2: Exact Duplicate Detection
+        const { exactClusters, nearCandidates, nextId: nextId1 } = this.findExactDuplicates(nodes, clusterCounter);
+        clusterCounter = nextId1;
+        // Phase 3: Near Duplicate Detection
+        const { nearClusters } = this.findNearDuplicates(nearCandidates, threshold, clusterCounter);
+        const allClusters = [...exactClusters, ...nearClusters];
+        // Phase 4, 5, 6: Process Clusters (Template-Heavy, Canonical, Representative)
+        this.processClusters(allClusters, graph, collapse);
+        // Final Edge Transfer if Collapsing
+        if (collapse) {
+            this.collapseEdges(graph);
+        }
+        return allClusters;
+    }
+    findExactDuplicates(nodes, startId) {
+        const exactMap = this.groupNodesByContentHash(nodes);
+        return this.createExactClusters(exactMap, startId);
+    }
+    groupNodesByContentHash(nodes) {
+        const exactMap = new Map();
+        for (const node of nodes) {
+            if (!node.contentHash || node.status !== 200)
+                continue;
+            let arr = exactMap.get(node.contentHash);
+            if (!arr) {
+                arr = [];
+                exactMap.set(node.contentHash, arr);
+            }
+            arr.push(node);
+        }
+        return exactMap;
+    }
+    createExactClusters(exactMap, startId) {
+        const exactClusters = [];
+        const nearCandidates = [];
+        let clusterCounter = startId;
+        for (const group of exactMap.values()) {
+            if (group.length > 1) {
+                const id = `cluster_exact_${clusterCounter++}`;
+                exactClusters.push({ id, type: 'exact', nodes: group, size: group.length });
+                for (const n of group) {
+                    n.duplicateClusterId = id;
+                    n.duplicateType = 'exact';
+                }
+            }
+            else {
+                nearCandidates.push(group[0]);
+            }
+        }
+        return { exactClusters, nearCandidates, nextId: clusterCounter };
+    }
+    findNearDuplicates(candidates, threshold, startId) {
+        const { bandsMaps, simhashes } = this.buildSimHashBuckets(candidates);
+        const { parent, involvedIndices } = this.findConnectedComponents(bandsMaps, simhashes, candidates.length, threshold);
+        return this.extractClusters(parent, involvedIndices, candidates, startId);
+    }
+    buildSimHashBuckets(candidates) {
+        const n = candidates.length;
+        const simhashes = new BigUint64Array(n);
+        const validIndices = [];
+        for (let i = 0; i < n; i++) {
+            if (candidates[i].simhash) {
+                simhashes[i] = BigInt(candidates[i].simhash);
+                validIndices.push(i);
+            }
+        }
+        const bandsMaps = Array.from({ length: SimHash.BANDS }, () => new Map());
+        for (const idx of validIndices) {
+            const bands = SimHash.getBands(simhashes[idx]);
+            for (let b = 0; b < SimHash.BANDS; b++) {
+                let arr = bandsMaps[b].get(bands[b]);
+                if (!arr) {
+                    arr = [];
+                    bandsMaps[b].set(bands[b], arr);
+                }
+                arr.push(idx);
+            }
+        }
+        return { bandsMaps, simhashes, validIndices };
+    }
+    findConnectedComponents(bandsMaps, simhashes, n, threshold) {
+        const parent = new Uint32Array(n);
+        const rank = new Uint8Array(n);
+        for (let i = 0; i < n; i++) {
+            parent[i] = i;
+            rank[i] = 0;
+        }
+        const find = (i) => {
+            let root = i;
+            while (parent[root] !== root) {
+                root = parent[root];
+            }
+            let curr = i;
+            while (curr !== root) {
+                const next = parent[curr];
+                parent[curr] = root;
+                curr = next;
+            }
+            return root;
+        };
+        const union = (i, j) => {
+            const rootI = find(i);
+            const rootJ = find(j);
+            if (rootI !== rootJ) {
+                const rankI = rank[rootI];
+                const rankJ = rank[rootJ];
+                if (rankI < rankJ) {
+                    parent[rootI] = rootJ;
+                }
+                else if (rankI > rankJ) {
+                    parent[rootJ] = rootI;
+                }
+                else {
+                    parent[rootJ] = rootI;
+                    rank[rootI]++;
+                }
+            }
+        };
+        const involvedIndices = new Set();
+        for (let b = 0; b < SimHash.BANDS; b++) {
+            for (const bucketIndices of bandsMaps[b].values()) {
+                if (bucketIndices.length < 2)
+                    continue;
+                for (let j = 0; j < bucketIndices.length; j++) {
+                    for (let k = j + 1; k < bucketIndices.length; k++) {
+                        const idx1 = bucketIndices[j];
+                        const idx2 = bucketIndices[k];
+                        const root1 = find(idx1);
+                        const root2 = find(idx2);
+                        if (root1 === root2)
+                            continue;
+                        const dist = SimHash.hammingDistance(simhashes[idx1], simhashes[idx2]);
+                        if (dist <= threshold) {
+                            union(root1, root2);
+                            involvedIndices.add(idx1);
+                            involvedIndices.add(idx2);
+                        }
+                    }
+                }
+            }
+        }
+        return { parent, involvedIndices };
+    }
+    extractClusters(parent, involvedIndices, candidates, startId) {
+        const nearClusters = [];
+        let clusterCounter = startId;
+        const find = (i) => {
+            let root = i;
+            while (parent[root] !== root) {
+                root = parent[root];
+            }
+            let curr = i;
+            while (curr !== root) {
+                const next = parent[curr];
+                parent[curr] = root;
+                curr = next;
+            }
+            return root;
+        };
+        const clusterMap = new Map();
+        for (const idx of involvedIndices) {
+            const root = find(idx);
+            let group = clusterMap.get(root);
+            if (!group) {
+                group = [];
+                clusterMap.set(root, group);
+            }
+            group.push(idx);
+        }
+        for (const groupIndices of clusterMap.values()) {
+            if (groupIndices.length > 1) {
+                const id = `cluster_near_${clusterCounter++}`;
+                const groupNodes = groupIndices.map(idx => candidates[idx]);
+                nearClusters.push({ id, type: 'near', nodes: groupNodes, size: groupNodes.length });
+                for (const n of groupNodes) {
+                    n.duplicateClusterId = id;
+                    n.duplicateType = 'near';
+                }
+            }
+        }
+        return { nearClusters, nextId: clusterCounter };
+    }
+    processClusters(clusters, graph, collapse) {
+        for (const cluster of clusters) {
+            this.processSingleCluster(cluster, graph, collapse);
+        }
+    }
+    processSingleCluster(cluster, graph, collapse) {
+        this.checkTemplateHeavy(cluster);
+        cluster.severity = this.calculateSeverity(cluster);
+        const representative = this.selectRepresentative(cluster);
+        cluster.representative = representative.url;
+        this.applyClusterToGraph(cluster, representative, graph, collapse);
+    }
+    checkTemplateHeavy(cluster) {
+        const avgRatio = cluster.nodes.reduce((sum, n) => sum + (n.uniqueTokenRatio || 0), 0) / cluster.nodes.length;
+        if (avgRatio < 0.3) {
+            cluster.type = 'template_heavy';
+            cluster.nodes.forEach(n => n.duplicateType = 'template_heavy');
+        }
+    }
+    calculateSeverity(cluster) {
+        const canonicals = new Set();
+        let hasMissing = false;
+        for (const n of cluster.nodes) {
+            if (!n.canonical)
+                hasMissing = true;
+            else
+                canonicals.add(n.canonical);
+        }
+        if (hasMissing || canonicals.size > 1) {
+            return 'high';
+        }
+        else if (cluster.type === 'near') {
+            return 'medium';
+        }
+        else {
+            return 'low';
+        }
+    }
+    selectRepresentative(cluster) {
+        const urlsInCluster = new Set(cluster.nodes.map(n => n.url));
+        const validCanonicals = cluster.nodes.filter(n => n.canonical && urlsInCluster.has(n.canonical) && n.url === n.canonical);
+        if (validCanonicals.length > 0) {
+            return validCanonicals[0];
+        }
+        return cluster.nodes.reduce((best, current) => {
+            if (current.inLinks > best.inLinks)
+                return current;
+            if (current.inLinks < best.inLinks)
+                return best;
+            if (current.url.length < best.url.length)
+                return current;
+            return best;
+        });
+    }
+    applyClusterToGraph(cluster, representative, graph, collapse) {
+        cluster.nodes.forEach(n => {
+            n.isClusterPrimary = n.url === representative.url;
+            n.isCollapsed = false;
+            n.collapseInto = undefined;
+        });
+        if (!graph.duplicateClusters) {
+            graph.duplicateClusters = [];
+        }
+        graph.duplicateClusters.push({
+            id: cluster.id,
+            type: cluster.type,
+            size: cluster.nodes.length,
+            representative: representative.url,
+            severity: cluster.severity
+        });
+        if (collapse) {
+            for (const n of cluster.nodes) {
+                if (n.url !== representative.url) {
+                    n.isCollapsed = true;
+                    n.collapseInto = representative.url;
+                }
+            }
+        }
+    }
+    collapseEdges(graph) {
+        const edges = graph.getEdges();
+        const updatedEdges = new Map();
+        for (const edge of edges) {
+            const targetNode = graph.nodes.get(edge.target);
+            if (!targetNode)
+                continue;
+            const actualSource = edge.source;
+            const actualTarget = targetNode.isCollapsed && targetNode.collapseInto ? targetNode.collapseInto : edge.target;
+            if (actualSource === actualTarget)
+                continue;
+            const edgeKey = Graph.getEdgeKey(actualSource, actualTarget);
+            const existingWeight = updatedEdges.get(edgeKey) || 0;
+            updatedEdges.set(edgeKey, Math.max(existingWeight, edge.weight));
+        }
+        graph.edges = updatedEdges;
+        // Re-calculate inLinks and outLinks
+        for (const node of graph.getNodes()) {
+            node.inLinks = 0;
+            node.outLinks = 0;
+        }
+        for (const edgeKey of updatedEdges.keys()) {
+            const { source: src, target: tgt } = Graph.parseEdgeKey(edgeKey);
+            const sn = graph.nodes.get(src);
+            const tn = graph.nodes.get(tgt);
+            if (sn)
+                sn.outLinks++;
+            if (tn)
+                tn.inLinks++;
+        }
+    }
+}