@crawlith/core 0.1.0

package/dist/graph/cluster.js

@@ -0,0 +1,173 @@
+import { SimHash } from './simhash.js';
+/**
+ * Detects content clusters using 64-bit SimHash and Hamming Distance.
+ * Uses band optimization to reduce O(n^2) comparisons.
+ */
+export function detectContentClusters(graph, threshold = 10, minSize = 3) {
+    const nodes = graph.getNodes().filter(n => n.simhash && n.status === 200);
+    if (nodes.length === 0)
+        return [];
+    const adjacency = new Map();
+    // Banding Optimization (4 bands of 16 bits)
+    // Note: For threshold > 3, this is a heuristic and may miss some pairs,
+    // but it dramatically reduces the search space as requested.
+    const bands = 4;
+    const bandWidth = 16;
+    const buckets = Array.from({ length: bands }, () => new Map());
+    for (const node of nodes) {
+        const hash = BigInt(node.simhash);
+        for (let b = 0; b < bands; b++) {
+            const bandValue = Number((hash >> BigInt(b * bandWidth)) & 0xffffn);
+            if (!buckets[b].has(bandValue)) {
+                buckets[b].set(bandValue, new Set());
+            }
+            buckets[b].get(bandValue).add(node.url);
+        }
+    }
+    const checkedPairs = new Set();
+    for (let b = 0; b < bands; b++) {
+        for (const bucket of buckets[b].values()) {
+            if (bucket.size < 2)
+                continue;
+            const bucketNodes = Array.from(bucket);
+            for (let i = 0; i < bucketNodes.length; i++) {
+                for (let j = i + 1; j < bucketNodes.length; j++) {
+                    const u1 = bucketNodes[i];
+                    const u2 = bucketNodes[j];
+                    if (u1 === u2)
+                        continue;
+                    const pairKey = u1 < u2 ? `${u1}|${u2}` : `${u2}|${u1}`;
+                    if (checkedPairs.has(pairKey))
+                        continue;
+                    checkedPairs.add(pairKey);
+                    const n1 = graph.nodes.get(u1);
+                    const n2 = graph.nodes.get(u2);
+                    const dist = SimHash.hammingDistance(BigInt(n1.simhash), BigInt(n2.simhash));
+                    if (dist <= threshold) {
+                        if (!adjacency.has(u1))
+                            adjacency.set(u1, new Set());
+                        if (!adjacency.has(u2))
+                            adjacency.set(u2, new Set());
+                        adjacency.get(u1).add(u2);
+                        adjacency.get(u2).add(u1);
+                    }
+                }
+            }
+        }
+    }
+    // Find connected components (Clusters)
+    const visited = new Set();
+    const clusters = [];
+    for (const node of nodes) {
+        if (visited.has(node.url))
+            continue;
+        const component = [];
+        const queue = [node.url];
+        visited.add(node.url);
+        while (queue.length > 0) {
+            const current = queue.shift();
+            component.push(current);
+            const neighbors = adjacency.get(current);
+            if (neighbors) {
+                for (const neighbor of neighbors) {
+                    if (!visited.has(neighbor)) {
+                        visited.add(neighbor);
+                        queue.push(neighbor);
+                    }
+                }
+            }
+        }
+        if (component.length >= minSize) {
+            clusters.push(component);
+        }
+    }
+    // Sort clusters by size (descending) then by primary URL (ascending) for deterministic IDs
+    clusters.sort((a, b) => {
+        if (b.length !== a.length)
+            return b.length - a.length;
+        const aPrimary = selectPrimaryUrl(a, graph);
+        const bPrimary = selectPrimaryUrl(b, graph);
+        return aPrimary.localeCompare(bPrimary);
+    });
+    const clusterInfos = [];
+    clusters.forEach((memberUrls, index) => {
+        const clusterId = index + 1;
+        const clusterNodes = memberUrls.map(url => graph.nodes.get(url));
+        for (const node of clusterNodes) {
+            node.clusterId = clusterId;
+        }
+        const primaryUrl = selectPrimaryUrl(memberUrls, graph);
+        const risk = calculateClusterRisk(clusterNodes);
+        const sharedPathPrefix = findSharedPathPrefix(memberUrls);
+        clusterInfos.push({
+            id: clusterId,
+            count: memberUrls.length,
+            primaryUrl,
+            risk,
+            sharedPathPrefix
+        });
+    });
+    graph.contentClusters = clusterInfos;
+    return clusterInfos;
+}
+/**
+ * Selects the primary URL for a cluster based on:
+ * 1. Highest PageRank
+ * 2. Shortest URL
+ * 3. Lexicographic fallback
+ */
+function selectPrimaryUrl(urls, graph) {
+    return urls.reduce((best, current) => {
+        const nBest = graph.nodes.get(best);
+        const nCurrent = graph.nodes.get(current);
+        if ((nCurrent.pageRank || 0) > (nBest.pageRank || 0))
+            return current;
+        if ((nCurrent.pageRank || 0) < (nBest.pageRank || 0))
+            return best;
+        if (current.length < best.length)
+            return current;
+        if (current.length > best.length)
+            return best;
+        return current.localeCompare(best) < 0 ? current : best;
+    });
+}
+/**
+ * Calculates cannibalization risk based on title and H1 similarity within the cluster.
+ */
+function calculateClusterRisk(nodes) {
+    // Logic: Check if there's significant overlap in Titles or H1s among cluster members.
+    // This is a heuristic as requested.
+    // Simplified heuristic: risk is based on cluster density and size
+    // Large clusters of highly similar content are high risk.
+    // Fallback to a safe categorization
+    if (nodes.length > 5)
+        return 'high';
+    if (nodes.length > 2)
+        return 'medium';
+    return 'low';
+}
+/**
+ * Finds the common path prefix among a set of URLs.
+ */
+function findSharedPathPrefix(urls) {
+    if (urls.length < 2)
+        return undefined;
+    try {
+        const paths = urls.map(u => new URL(u).pathname.split('/').filter(Boolean));
+        const first = paths[0];
+        const common = [];
+        for (let i = 0; i < first.length; i++) {
+            const segment = first[i];
+            if (paths.every(p => p[i] === segment)) {
+                common.push(segment);
+            }
+            else {
+                break;
+            }
+        }
+        return common.length > 0 ? '/' + common.join('/') : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}

package/dist/graph/duplicate.d.ts

@@ -0,0 +1,10 @@
+import { Graph } from './graph.js';
+export interface DuplicateOptions {
+    collapse?: boolean;
+    simhashThreshold?: number;
+}
+/**
+ * Detects exact and near duplicates, identifies canonical conflicts,
+ * and performs non-destructive collapse of edges.
+ */
+export declare function detectDuplicates(graph: Graph, options?: DuplicateOptions): void;

package/dist/graph/duplicate.js

@@ -0,0 +1,251 @@
+import { SimHash } from './simhash.js';
+/**
+ * Detects exact and near duplicates, identifies canonical conflicts,
+ * and performs non-destructive collapse of edges.
+ */
+export function detectDuplicates(graph, options = {}) {
+    const collapse = options.collapse !== false; // Default to true
+    const threshold = options.simhashThreshold ?? 3;
+    const exactClusters = [];
+    const nearClusters = [];
+    const nodes = graph.getNodes();
+    // Phase 1 & 2: Exact Duplicate Detection
+    const exactMap = new Map();
+    for (const node of nodes) {
+        if (!node.contentHash || node.status !== 200)
+            continue;
+        // Safety check: if there's no soft404 signal (soft404 is handled elsewhere, but just filter 200 OKs)
+        let arr = exactMap.get(node.contentHash);
+        if (!arr) {
+            arr = [];
+            exactMap.set(node.contentHash, arr);
+        }
+        arr.push(node);
+    }
+    // Nodes that are NOT part of an exact duplicate group are candidates for near duplicate checks
+    const nearCandidates = [];
+    let clusterCounter = 1;
+    for (const [_hash, group] of exactMap.entries()) {
+        if (group.length > 1) {
+            const id = `cluster_exact_${clusterCounter++}`;
+            exactClusters.push({ id, type: 'exact', nodes: group });
+            // Mark nodes
+            for (const n of group) {
+                n.duplicateClusterId = id;
+                n.duplicateType = 'exact';
+            }
+        }
+        else {
+            nearCandidates.push(group[0]);
+        }
+    }
+    // Phase 3: Near Duplicate Detection (SimHash with Bands)
+    // 64-bit simhash -> split into 4 bands of 16 bits.
+    const bandsMaps = [
+        new Map(),
+        new Map(),
+        new Map(),
+        new Map()
+    ];
+    for (const node of nearCandidates) {
+        if (!node.simhash)
+            continue;
+        const simhash = BigInt(node.simhash);
+        // Extract 16 bit bands
+        const b0 = Number(simhash & 0xffffn);
+        const b1 = Number((simhash >> 16n) & 0xffffn);
+        const b2 = Number((simhash >> 32n) & 0xffffn);
+        const b3 = Number((simhash >> 48n) & 0xffffn);
+        const bands = [b0, b1, b2, b3];
+        for (let i = 0; i < 4; i++) {
+            let arr = bandsMaps[i].get(bands[i]);
+            if (!arr) {
+                arr = [];
+                bandsMaps[i].set(bands[i], arr);
+            }
+            arr.push(node);
+        }
+    }
+    // Find candidate pairs
+    const nearGroupMap = new Map(); // node.url -> cluster set
+    const checkedPairs = new Set();
+    for (let i = 0; i < 4; i++) {
+        for (const [_bandVal, bucketNodes] of bandsMaps[i].entries()) {
+            if (bucketNodes.length < 2)
+                continue; // nothing to compare
+            // Compare all nodes in this bucket
+            for (let j = 0; j < bucketNodes.length; j++) {
+                for (let k = j + 1; k < bucketNodes.length; k++) {
+                    const n1 = bucketNodes[j];
+                    const n2 = bucketNodes[k];
+                    // Ensure n1 < n2 lexicographically to avoid duplicate pairs
+                    const [a, b] = n1.url < n2.url ? [n1, n2] : [n2, n1];
+                    const pairKey = `${a.url}|${b.url}`;
+                    if (checkedPairs.has(pairKey))
+                        continue;
+                    checkedPairs.add(pairKey);
+                    const dist = SimHash.hammingDistance(BigInt(a.simhash), BigInt(b.simhash));
+                    if (dist <= threshold) {
+                        // They are near duplicates. 
+                        // Find or create their cluster set using union-find or reference propagation
+                        const setA = nearGroupMap.get(a.url);
+                        const setB = nearGroupMap.get(b.url);
+                        if (!setA && !setB) {
+                            const newSet = new Set([a, b]);
+                            nearGroupMap.set(a.url, newSet);
+                            nearGroupMap.set(b.url, newSet);
+                        }
+                        else if (setA && !setB) {
+                            setA.add(b);
+                            nearGroupMap.set(b.url, setA);
+                        }
+                        else if (setB && !setA) {
+                            setB.add(a);
+                            nearGroupMap.set(a.url, setB);
+                        }
+                        else if (setA && setB && setA !== setB) {
+                            // Merge sets
+                            for (const node of setB) {
+                                setA.add(node);
+                                nearGroupMap.set(node.url, setA);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    // Compile near duplicate clusters (deduplicated by Set reference)
+    const uniqueNearSets = new Set();
+    for (const group of nearGroupMap.values()) {
+        uniqueNearSets.add(group);
+    }
+    for (const groupSet of uniqueNearSets) {
+        if (groupSet.size > 1) {
+            const id = `cluster_near_${clusterCounter++}`;
+            const groupArr = Array.from(groupSet);
+            nearClusters.push({ id, type: 'near', nodes: groupArr });
+            for (const n of groupArr) {
+                n.duplicateClusterId = id;
+                n.duplicateType = 'near';
+            }
+        }
+    }
+    const allClusters = [...exactClusters, ...nearClusters];
+    // Phase 4: Template-Heavy Detection
+    // Mark classes as 'template_heavy' if ratio < 0.3
+    for (const cluster of allClusters) {
+        const avgRatio = cluster.nodes.reduce((sum, n) => sum + (n.uniqueTokenRatio || 0), 0) / cluster.nodes.length;
+        if (avgRatio < 0.3) {
+            cluster.type = 'template_heavy';
+            cluster.nodes.forEach(n => n.duplicateType = 'template_heavy');
+        }
+    }
+    // Phase 5: Canonical Conflict & Representative Selection
+    for (const cluster of allClusters) {
+        const canonicals = new Set();
+        let hasMissing = false;
+        for (const n of cluster.nodes) {
+            if (!n.canonical)
+                hasMissing = true;
+            // We compare full absolute canonical URLs (assuming they are normalized during crawl)
+            else
+                canonicals.add(n.canonical);
+        }
+        if (hasMissing || canonicals.size > 1) {
+            cluster.severity = 'high';
+        }
+        else if (cluster.type === 'near') {
+            cluster.severity = 'medium';
+        }
+        else {
+            cluster.severity = 'low';
+        }
+        // Phase 6: Select Representative
+        // 1. Valid Canonical target in cluster
+        // 2. Highest internal in-degree
+        // 3. Shortest URL
+        // 4. First discovered (relying on array order, which is from BFS map roughly)
+        let representativeNode = cluster.nodes[0];
+        // Evaluate best rep
+        const urlsInCluster = new Set(cluster.nodes.map(n => n.url));
+        const validCanonicals = cluster.nodes.filter(n => n.canonical && urlsInCluster.has(n.canonical) && n.url === n.canonical);
+        if (validCanonicals.length > 0) {
+            representativeNode = validCanonicals[0]; // If multiple, just pick first matching self
+        }
+        else {
+            representativeNode = cluster.nodes.reduce((best, current) => {
+                if (current.inLinks > best.inLinks)
+                    return current;
+                if (current.inLinks < best.inLinks)
+                    return best;
+                if (current.url.length < best.url.length)
+                    return current;
+                return best;
+            });
+        }
+        cluster.representative = representativeNode.url;
+        cluster.nodes.forEach(n => {
+            n.isClusterPrimary = n.url === representativeNode.url;
+            n.isCollapsed = false; // default for JSON
+            n.collapseInto = undefined;
+        });
+        // Push to Graph's final cluster list
+        graph.duplicateClusters.push({
+            id: cluster.id,
+            type: cluster.type,
+            size: cluster.nodes.length,
+            representative: representativeNode.url,
+            severity: cluster.severity
+        });
+        // Controlled Collapse
+        if (collapse) {
+            for (const n of cluster.nodes) {
+                if (n.url !== representativeNode.url) {
+                    n.isCollapsed = true;
+                    n.collapseInto = representativeNode.url;
+                }
+            }
+        }
+    }
+    // Final Edge Transfer if Collapsing
+    if (collapse) {
+        const edges = graph.getEdges();
+        const updatedEdges = new Map();
+        for (const edge of edges) {
+            const sourceNode = graph.nodes.get(edge.source);
+            const targetNode = graph.nodes.get(edge.target);
+            if (!sourceNode || !targetNode)
+                continue;
+            // We do NOT modify source structure for out-bound edges of collapsed nodes? 
+            // Spec: "Ignore edges from collapsed nodes. Transfer inbound edges to representative."
+            // Actually, if a node links TO a collapsed node, we repoint the edge to the representative.
+            // If a collapsed node links to X, we ignore it (PageRank will filter it out).
+            const actualSource = edge.source;
+            // repoint target
+            const actualTarget = targetNode.isCollapsed && targetNode.collapseInto ? targetNode.collapseInto : edge.target;
+            // Skip self-referential edges caused by repointing
+            if (actualSource === actualTarget)
+                continue;
+            const edgeKey = `${actualSource}|${actualTarget}`;
+            const existingWeight = updatedEdges.get(edgeKey) || 0;
+            updatedEdges.set(edgeKey, Math.max(existingWeight, edge.weight)); // deduplicate
+        }
+        // Update graph edges in-place
+        graph.edges = updatedEdges;
+        // Re-calculate inLinks and outLinks based on collapsed edges
+        for (const node of graph.getNodes()) {
+            node.inLinks = 0;
+            node.outLinks = 0;
+        }
+        for (const [edgeKey, _weight] of updatedEdges.entries()) {
+            const [src, tgt] = edgeKey.split('|');
+            const sn = graph.nodes.get(src);
+            const tn = graph.nodes.get(tgt);
+            if (sn)
+                sn.outLinks++;
+            if (tn)
+                tn.inLinks++;
+        }
+    }
+}

package/dist/graph/graph.d.ts

@@ -0,0 +1,103 @@
+export interface GraphNode {
+    url: string;
+    depth: number;
+    inLinks: number;
+    outLinks: number;
+    status: number;
+    canonical?: string;
+    noindex?: boolean;
+    nofollow?: boolean;
+    brokenLinks?: string[];
+    redirectChain?: string[];
+    incrementalStatus?: 'new' | 'changed' | 'unchanged' | 'deleted';
+    etag?: string;
+    lastModified?: string;
+    contentHash?: string;
+    html?: string;
+    pageRank?: number;
+    pageRankScore?: number;
+    authorityScore?: number;
+    hubScore?: number;
+    duplicateClusterId?: string;
+    duplicateType?: 'exact' | 'near' | 'template_heavy' | 'none';
+    isClusterPrimary?: boolean;
+    isCollapsed?: boolean;
+    collapseInto?: string;
+    simhash?: string;
+    uniqueTokenRatio?: number;
+    soft404Score?: number;
+    soft404Signals?: string[];
+    crawlTrapFlag?: boolean;
+    crawlTrapRisk?: number;
+    trapType?: string;
+    securityError?: string;
+    retries?: number;
+    clusterId?: number;
+    bytesReceived?: number;
+    linkRole?: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
+}
+export interface GraphEdge {
+    source: string;
+    target: string;
+    weight: number;
+}
+export interface ClusterInfo {
+    id: number;
+    count: number;
+    primaryUrl: string;
+    risk: 'low' | 'medium' | 'high';
+    sharedPathPrefix?: string;
+}
+export interface CrawlStats {
+    pagesFetched: number;
+    pagesCached: number;
+    pagesSkipped: number;
+    totalFound: number;
+}
+export declare class Graph {
+    nodes: Map<string, GraphNode>;
+    edges: Map<string, number>;
+    limitReached: boolean;
+    sessionStats: CrawlStats;
+    trapClusters: {
+        pattern: string;
+        type: string;
+        count: number;
+    }[];
+    duplicateClusters: {
+        id: string;
+        type: 'exact' | 'near' | 'template_heavy';
+        size: number;
+        representative: string;
+        severity: 'low' | 'medium' | 'high';
+    }[];
+    contentClusters: ClusterInfo[];
+    /**
+     * Adds a node to the graph if it doesn't exist.
+     * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
+     * Depth is only set on creation (BFS guarantees shortest path first).
+     */
+    addNode(url: string, depth: number, status?: number): void;
+    updateNodeData(url: string, data: Partial<GraphNode>): void;
+    /**
+     * Adds a directed edge between two nodes.
+     * Both nodes must exist in the graph.
+     * Updates inLinks and outLinks counts.
+     */
+    addEdge(source: string, target: string, weight?: number): void;
+    getNodes(): GraphNode[];
+    getEdges(): GraphEdge[];
+    toJSON(): {
+        nodes: GraphNode[];
+        edges: GraphEdge[];
+        duplicateClusters: {
+            id: string;
+            type: "exact" | "near" | "template_heavy";
+            size: number;
+            representative: string;
+            severity: "low" | "medium" | "high";
+        }[];
+        contentClusters: ClusterInfo[];
+    };
+    static fromJSON(json: any): Graph;
+}

package/dist/graph/graph.js

@@ -0,0 +1,106 @@
+export class Graph {
+    nodes = new Map();
+    // Using string "source|target" to ensure uniqueness efficiently. Mapping to weight.
+    edges = new Map();
+    limitReached = false;
+    sessionStats = {
+        pagesFetched: 0,
+        pagesCached: 0,
+        pagesSkipped: 0,
+        totalFound: 0
+    };
+    trapClusters = [];
+    duplicateClusters = [];
+    contentClusters = [];
+    /**
+     * Adds a node to the graph if it doesn't exist.
+     * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
+     * Depth is only set on creation (BFS guarantees shortest path first).
+     */
+    addNode(url, depth, status = 0) {
+        const existing = this.nodes.get(url);
+        if (!existing) {
+            this.nodes.set(url, {
+                url,
+                depth,
+                status,
+                inLinks: 0,
+                outLinks: 0
+            });
+        }
+        else {
+            // Update status if we have a real one now (e.g. was 0/pending, now crawled)
+            if (status !== 0) {
+                existing.status = status;
+            }
+        }
+    }
+    updateNodeData(url, data) {
+        const existing = this.nodes.get(url);
+        if (existing) {
+            Object.assign(existing, data);
+        }
+    }
+    /**
+     * Adds a directed edge between two nodes.
+     * Both nodes must exist in the graph.
+     * Updates inLinks and outLinks counts.
+     */
+    addEdge(source, target, weight = 1.0) {
+        const sourceNode = this.nodes.get(source);
+        const targetNode = this.nodes.get(target);
+        if (sourceNode && targetNode) {
+            const edgeKey = `${source}|${target}`;
+            if (!this.edges.has(edgeKey)) {
+                this.edges.set(edgeKey, weight);
+                sourceNode.outLinks++;
+                targetNode.inLinks++;
+            }
+            else {
+                // If edge exists, keep highest weight (or could sum, but usually we just want the 'best' relationship)
+                const currentWeight = this.edges.get(edgeKey) || 0;
+                if (weight > currentWeight) {
+                    this.edges.set(edgeKey, weight);
+                }
+            }
+        }
+    }
+    getNodes() {
+        return Array.from(this.nodes.values());
+    }
+    getEdges() {
+        return Array.from(this.edges.entries()).map(([edge, weight]) => {
+            const [source, target] = edge.split('|');
+            return { source, target, weight };
+        });
+    }
+    toJSON() {
+        return {
+            nodes: this.getNodes(),
+            edges: this.getEdges(),
+            duplicateClusters: this.duplicateClusters,
+            contentClusters: this.contentClusters
+        };
+    }
+    static fromJSON(json) {
+        const graph = new Graph();
+        if (json.nodes) {
+            for (const node of json.nodes) {
+                graph.nodes.set(node.url, { ...node });
+            }
+        }
+        if (json.edges) {
+            for (const edge of json.edges) {
+                const key = `${edge.source}|${edge.target}`;
+                graph.edges.set(key, edge.weight || 1.0);
+            }
+        }
+        if (json.duplicateClusters) {
+            graph.duplicateClusters = json.duplicateClusters;
+        }
+        if (json.contentClusters) {
+            graph.contentClusters = json.contentClusters;
+        }
+        return graph;
+    }
+}

package/dist/graph/metrics.d.ts

@@ -0,0 +1,29 @@
+import { Graph } from './graph.js';
+export interface Metrics {
+    totalPages: number;
+    totalEdges: number;
+    orphanPages: string[];
+    nearOrphans: string[];
+    deepPages: string[];
+    topAuthorityPages: {
+        url: string;
+        authority: number;
+    }[];
+    averageOutDegree: number;
+    maxDepthFound: number;
+    crawlEfficiencyScore: number;
+    averageDepth: number;
+    structuralEntropy: number;
+    topPageRankPages: {
+        url: string;
+        score: number;
+    }[];
+    limitReached: boolean;
+    sessionStats?: {
+        pagesFetched: number;
+        pagesCached: number;
+        pagesSkipped: number;
+        totalFound: number;
+    };
+}
+export declare function calculateMetrics(graph: Graph, _maxDepth: number): Metrics;