@crawlith/core 0.1.0 → 0.1.2

package/src/graph/cluster.ts

@@ -1,192 +0,0 @@
-import { Graph, GraphNode, ClusterInfo } from './graph.js';
-import { SimHash } from './simhash.js';
-
-/**
- * Detects content clusters using 64-bit SimHash and Hamming Distance.
- * Uses band optimization to reduce O(n^2) comparisons.
- */
-export function detectContentClusters(
-    graph: Graph,
-    threshold: number = 10,
-    minSize: number = 3
-): ClusterInfo[] {
-    const nodes = graph.getNodes().filter(n => n.simhash && n.status === 200);
-    if (nodes.length === 0) return [];
-
-    const adjacency = new Map<string, Set<string>>();
-
-    // Banding Optimization (4 bands of 16 bits)
-    // Note: For threshold > 3, this is a heuristic and may miss some pairs,
-    // but it dramatically reduces the search space as requested.
-    const bands = 4;
-    const bandWidth = 16;
-    const buckets: Map<number, Set<string>>[] = Array.from({ length: bands }, () => new Map());
-
-    for (const node of nodes) {
-        const hash = BigInt(node.simhash!);
-        for (let b = 0; b < bands; b++) {
-            const bandValue = Number((hash >> BigInt(b * bandWidth)) & 0xFFFFn);
-            if (!buckets[b].has(bandValue)) {
-                buckets[b].set(bandValue, new Set());
-            }
-            buckets[b].get(bandValue)!.add(node.url);
-        }
-    }
-
-    const checkedPairs = new Set<string>();
-
-    for (let b = 0; b < bands; b++) {
-        for (const bucket of buckets[b].values()) {
-            if (bucket.size < 2) continue;
-            const bucketNodes = Array.from(bucket);
-            for (let i = 0; i < bucketNodes.length; i++) {
-                for (let j = i + 1; j < bucketNodes.length; j++) {
-                    const u1 = bucketNodes[i];
-                    const u2 = bucketNodes[j];
-                    if (u1 === u2) continue;
-
-                    const pairKey = u1 < u2 ? `${u1}|${u2}` : `${u2}|${u1}`;
-                    if (checkedPairs.has(pairKey)) continue;
-                    checkedPairs.add(pairKey);
-
-                    const n1 = graph.nodes.get(u1)!;
-                    const n2 = graph.nodes.get(u2)!;
-
-                    const dist = SimHash.hammingDistance(BigInt(n1.simhash!), BigInt(n2.simhash!));
-                    if (dist <= threshold) {
-                        if (!adjacency.has(u1)) adjacency.set(u1, new Set());
-                        if (!adjacency.has(u2)) adjacency.set(u2, new Set());
-                        adjacency.get(u1)!.add(u2);
-                        adjacency.get(u2)!.add(u1);
-                    }
-                }
-            }
-        }
-    }
-
-    // Find connected components (Clusters)
-    const visited = new Set<string>();
-    const clusters: string[][] = [];
-
-    for (const node of nodes) {
-        if (visited.has(node.url)) continue;
-
-        const component: string[] = [];
-        const queue = [node.url];
-        visited.add(node.url);
-
-        while (queue.length > 0) {
-            const current = queue.shift()!;
-            component.push(current);
-
-            const neighbors = adjacency.get(current);
-            if (neighbors) {
-                for (const neighbor of neighbors) {
-                    if (!visited.has(neighbor)) {
-                        visited.add(neighbor);
-                        queue.push(neighbor);
-                    }
-                }
-            }
-        }
-
-        if (component.length >= minSize) {
-            clusters.push(component);
-        }
-    }
-
-    // Sort clusters by size (descending) then by primary URL (ascending) for deterministic IDs
-    clusters.sort((a, b) => {
-        if (b.length !== a.length) return b.length - a.length;
-        const aPrimary = selectPrimaryUrl(a, graph);
-        const bPrimary = selectPrimaryUrl(b, graph);
-        return aPrimary.localeCompare(bPrimary);
-    });
-
-    const clusterInfos: ClusterInfo[] = [];
-    clusters.forEach((memberUrls, index) => {
-        const clusterId = index + 1;
-        const clusterNodes = memberUrls.map(url => graph.nodes.get(url)!);
-
-        for (const node of clusterNodes) {
-            node.clusterId = clusterId;
-        }
-
-        const primaryUrl = selectPrimaryUrl(memberUrls, graph);
-        const risk = calculateClusterRisk(clusterNodes);
-        const sharedPathPrefix = findSharedPathPrefix(memberUrls);
-
-        clusterInfos.push({
-            id: clusterId,
-            count: memberUrls.length,
-            primaryUrl,
-            risk,
-            sharedPathPrefix
-        });
-    });
-
-    graph.contentClusters = clusterInfos;
-    return clusterInfos;
-}
-
-/**
- * Selects the primary URL for a cluster based on:
- * 1. Highest PageRank
- * 2. Shortest URL
- * 3. Lexicographic fallback
- */
-function selectPrimaryUrl(urls: string[], graph: Graph): string {
-    return urls.reduce((best, current) => {
-        const nBest = graph.nodes.get(best)!;
-        const nCurrent = graph.nodes.get(current)!;
-
-        if ((nCurrent.pageRank || 0) > (nBest.pageRank || 0)) return current;
-        if ((nCurrent.pageRank || 0) < (nBest.pageRank || 0)) return best;
-
-        if (current.length < best.length) return current;
-        if (current.length > best.length) return best;
-
-        return current.localeCompare(best) < 0 ? current : best;
-    });
-}
-
-/**
- * Calculates cannibalization risk based on title and H1 similarity within the cluster.
- */
-function calculateClusterRisk(nodes: GraphNode[]): 'low' | 'medium' | 'high' {
-    // Logic: Check if there's significant overlap in Titles or H1s among cluster members.
-    // This is a heuristic as requested.
-    // Simplified heuristic: risk is based on cluster density and size
-    // Large clusters of highly similar content are high risk.
-
-    // Fallback to a safe categorization
-    if (nodes.length > 5) return 'high';
-    if (nodes.length > 2) return 'medium';
-    return 'low';
-}
-
-/**
- * Finds the common path prefix among a set of URLs.
- */
-function findSharedPathPrefix(urls: string[]): string | undefined {
-    if (urls.length < 2) return undefined;
-
-    try {
-        const paths = urls.map(u => new URL(u).pathname.split('/').filter(Boolean));
-        const first = paths[0];
-        const common: string[] = [];
-
-        for (let i = 0; i < first.length; i++) {
-            const segment = first[i];
-            if (paths.every(p => p[i] === segment)) {
-                common.push(segment);
-            } else {
-                break;
-            }
-        }
-
-        return common.length > 0 ? '/' + common.join('/') : undefined;
-    } catch {
-        return undefined;
-    }
-}

package/src/graph/duplicate.ts

@@ -1,286 +0,0 @@
-import { Graph, GraphNode } from './graph.js';
-import { SimHash } from './simhash.js';
-
-export interface DuplicateOptions {
-    collapse?: boolean;
-    simhashThreshold?: number; // Hamming distance threshold (default: 3)
-}
-
-interface DuplicateCluster {
-    id: string;
-    type: 'exact' | 'near' | 'template_heavy';
-    nodes: GraphNode[];
-    representative?: string;
-    severity?: 'low' | 'medium' | 'high';
-}
-
-/**
- * Detects exact and near duplicates, identifies canonical conflicts,
- * and performs non-destructive collapse of edges.
- */
-export function detectDuplicates(graph: Graph, options: DuplicateOptions = {}) {
-    const collapse = options.collapse !== false; // Default to true
-    const threshold = options.simhashThreshold ?? 3;
-
-    const exactClusters: DuplicateCluster[] = [];
-    const nearClusters: DuplicateCluster[] = [];
-
-    const nodes = graph.getNodes();
-
-    // Phase 1 & 2: Exact Duplicate Detection
-    const exactMap = new Map<string, GraphNode[]>();
-    for (const node of nodes) {
-        if (!node.contentHash || node.status !== 200) continue;
-
-        // Safety check: if there's no soft404 signal (soft404 is handled elsewhere, but just filter 200 OKs)
-        let arr = exactMap.get(node.contentHash);
-        if (!arr) {
-            arr = [];
-            exactMap.set(node.contentHash, arr);
-        }
-        arr.push(node);
-    }
-
-    // Nodes that are NOT part of an exact duplicate group are candidates for near duplicate checks
-    const nearCandidates: GraphNode[] = [];
-    let clusterCounter = 1;
-
-    for (const [_hash, group] of exactMap.entries()) {
-        if (group.length > 1) {
-            const id = `cluster_exact_${clusterCounter++}`;
-            exactClusters.push({ id, type: 'exact', nodes: group });
-            // Mark nodes
-            for (const n of group) {
-                n.duplicateClusterId = id;
-                n.duplicateType = 'exact';
-            }
-        } else {
-            nearCandidates.push(group[0]);
-        }
-    }
-
-    // Phase 3: Near Duplicate Detection (SimHash with Bands)
-    // 64-bit simhash -> split into 4 bands of 16 bits.
-    const bandsMaps = [
-        new Map<number, GraphNode[]>(),
-        new Map<number, GraphNode[]>(),
-        new Map<number, GraphNode[]>(),
-        new Map<number, GraphNode[]>()
-    ];
-
-    for (const node of nearCandidates) {
-        if (!node.simhash) continue;
-        const simhash = BigInt(node.simhash);
-
-        // Extract 16 bit bands
-        const b0 = Number(simhash & 0xFFFFn);
-        const b1 = Number((simhash >> 16n) & 0xFFFFn);
-        const b2 = Number((simhash >> 32n) & 0xFFFFn);
-        const b3 = Number((simhash >> 48n) & 0xFFFFn);
-
-        const bands = [b0, b1, b2, b3];
-        for (let i = 0; i < 4; i++) {
-            let arr = bandsMaps[i].get(bands[i]);
-            if (!arr) {
-                arr = [];
-                bandsMaps[i].set(bands[i], arr);
-            }
-            arr.push(node);
-        }
-    }
-
-    // Find candidate pairs
-    const nearGroupMap = new Map<string, Set<GraphNode>>(); // node.url -> cluster set
-    const checkedPairs = new Set<string>();
-
-    for (let i = 0; i < 4; i++) {
-        for (const [_bandVal, bucketNodes] of bandsMaps[i].entries()) {
-            if (bucketNodes.length < 2) continue; // nothing to compare
-
-            // Compare all nodes in this bucket
-            for (let j = 0; j < bucketNodes.length; j++) {
-                for (let k = j + 1; k < bucketNodes.length; k++) {
-                    const n1 = bucketNodes[j];
-                    const n2 = bucketNodes[k];
-
-                    // Ensure n1 < n2 lexicographically to avoid duplicate pairs
-                    const [a, b] = n1.url < n2.url ? [n1, n2] : [n2, n1];
-                    const pairKey = `${a.url}|${b.url}`;
-
-                    if (checkedPairs.has(pairKey)) continue;
-                    checkedPairs.add(pairKey);
-
-                    const dist = SimHash.hammingDistance(BigInt(a.simhash!), BigInt(b.simhash!));
-                    if (dist <= threshold) {
-                        // They are near duplicates. 
-                        // Find or create their cluster set using union-find or reference propagation
-                        const setA = nearGroupMap.get(a.url);
-                        const setB = nearGroupMap.get(b.url);
-
-                        if (!setA && !setB) {
-                            const newSet = new Set<GraphNode>([a, b]);
-                            nearGroupMap.set(a.url, newSet);
-                            nearGroupMap.set(b.url, newSet);
-                        } else if (setA && !setB) {
-                            setA.add(b);
-                            nearGroupMap.set(b.url, setA);
-                        } else if (setB && !setA) {
-                            setB.add(a);
-                            nearGroupMap.set(a.url, setB);
-                        } else if (setA && setB && setA !== setB) {
-                            // Merge sets
-                            for (const node of setB) {
-                                setA.add(node);
-                                nearGroupMap.set(node.url, setA);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    // Compile near duplicate clusters (deduplicated by Set reference)
-    const uniqueNearSets = new Set<Set<GraphNode>>();
-    for (const group of nearGroupMap.values()) {
-        uniqueNearSets.add(group);
-    }
-
-    for (const groupSet of uniqueNearSets) {
-        if (groupSet.size > 1) {
-            const id = `cluster_near_${clusterCounter++}`;
-            const groupArr = Array.from(groupSet);
-            nearClusters.push({ id, type: 'near', nodes: groupArr });
-            for (const n of groupArr) {
-                n.duplicateClusterId = id;
-                n.duplicateType = 'near';
-            }
-        }
-    }
-
-    const allClusters = [...exactClusters, ...nearClusters];
-
-    // Phase 4: Template-Heavy Detection
-    // Mark classes as 'template_heavy' if ratio < 0.3
-    for (const cluster of allClusters) {
-        const avgRatio = cluster.nodes.reduce((sum, n) => sum + (n.uniqueTokenRatio || 0), 0) / cluster.nodes.length;
-        if (avgRatio < 0.3) {
-            cluster.type = 'template_heavy';
-            cluster.nodes.forEach(n => n.duplicateType = 'template_heavy');
-        }
-    }
-
-    // Phase 5: Canonical Conflict & Representative Selection
-    for (const cluster of allClusters) {
-        const canonicals = new Set<string>();
-        let hasMissing = false;
-
-        for (const n of cluster.nodes) {
-            if (!n.canonical) hasMissing = true;
-            // We compare full absolute canonical URLs (assuming they are normalized during crawl)
-            else canonicals.add(n.canonical);
-        }
-
-        if (hasMissing || canonicals.size > 1) {
-            cluster.severity = 'high';
-        } else if (cluster.type === 'near') {
-            cluster.severity = 'medium';
-        } else {
-            cluster.severity = 'low';
-        }
-
-        // Phase 6: Select Representative
-        // 1. Valid Canonical target in cluster
-        // 2. Highest internal in-degree
-        // 3. Shortest URL
-        // 4. First discovered (relying on array order, which is from BFS map roughly)
-        let representativeNode = cluster.nodes[0];
-
-        // Evaluate best rep
-        const urlsInCluster = new Set(cluster.nodes.map(n => n.url));
-        const validCanonicals = cluster.nodes.filter(n => n.canonical && urlsInCluster.has(n.canonical) && n.url === n.canonical);
-
-        if (validCanonicals.length > 0) {
-            representativeNode = validCanonicals[0]; // If multiple, just pick first matching self
-        } else {
-            representativeNode = cluster.nodes.reduce((best, current) => {
-                if (current.inLinks > best.inLinks) return current;
-                if (current.inLinks < best.inLinks) return best;
-                if (current.url.length < best.url.length) return current;
-                return best;
-            });
-        }
-
-        cluster.representative = representativeNode.url;
-
-        cluster.nodes.forEach(n => {
-            n.isClusterPrimary = n.url === representativeNode.url;
-            n.isCollapsed = false; // default for JSON
-            n.collapseInto = undefined;
-        });
-
-        // Push to Graph's final cluster list
-        graph.duplicateClusters.push({
-            id: cluster.id,
-            type: cluster.type,
-            size: cluster.nodes.length,
-            representative: representativeNode.url,
-            severity: cluster.severity!
-        });
-
-        // Controlled Collapse
-        if (collapse) {
-            for (const n of cluster.nodes) {
-                if (n.url !== representativeNode.url) {
-                    n.isCollapsed = true;
-                    n.collapseInto = representativeNode.url;
-                }
-            }
-        }
-    }
-
-    // Final Edge Transfer if Collapsing
-    if (collapse) {
-        const edges = graph.getEdges();
-        const updatedEdges = new Map<string, number>();
-
-        for (const edge of edges) {
-            const sourceNode = graph.nodes.get(edge.source);
-            const targetNode = graph.nodes.get(edge.target);
-
-            if (!sourceNode || !targetNode) continue;
-
-            // We do NOT modify source structure for out-bound edges of collapsed nodes? 
-            // Spec: "Ignore edges from collapsed nodes. Transfer inbound edges to representative."
-            // Actually, if a node links TO a collapsed node, we repoint the edge to the representative.
-            // If a collapsed node links to X, we ignore it (PageRank will filter it out).
-
-            const actualSource = edge.source;
-            // repoint target
-            const actualTarget = targetNode.isCollapsed && targetNode.collapseInto ? targetNode.collapseInto : edge.target;
-
-            // Skip self-referential edges caused by repointing
-            if (actualSource === actualTarget) continue;
-
-            const edgeKey = `${actualSource}|${actualTarget}`;
-            const existingWeight = updatedEdges.get(edgeKey) || 0;
-            updatedEdges.set(edgeKey, Math.max(existingWeight, edge.weight)); // deduplicate
-        }
-
-        // Update graph edges in-place
-        graph.edges = updatedEdges;
-
-        // Re-calculate inLinks and outLinks based on collapsed edges
-        for (const node of graph.getNodes()) {
-            node.inLinks = 0;
-            node.outLinks = 0;
-        }
-        for (const [edgeKey, _weight] of updatedEdges.entries()) {
-            const [src, tgt] = edgeKey.split('|');
-            const sn = graph.nodes.get(src);
-            const tn = graph.nodes.get(tgt);
-            if (sn) sn.outLinks++;
-            if (tn) tn.inLinks++;
-        }
-    }
-}

package/src/graph/graph.ts

@@ -1,172 +0,0 @@
-export interface GraphNode {
-  url: string;
-  depth: number;
-  inLinks: number;
-  outLinks: number;
-  status: number;
-  canonical?: string;
-  noindex?: boolean;
-  nofollow?: boolean;
-  brokenLinks?: string[];
-  redirectChain?: string[];
-  incrementalStatus?: 'new' | 'changed' | 'unchanged' | 'deleted';
-  etag?: string;
-  lastModified?: string;
-  contentHash?: string;
-  html?: string;
-  pageRank?: number;
-  pageRankScore?: number;
-  authorityScore?: number;
-  hubScore?: number;
-  duplicateClusterId?: string;
-  duplicateType?: 'exact' | 'near' | 'template_heavy' | 'none';
-  isClusterPrimary?: boolean;
-  isCollapsed?: boolean;
-  collapseInto?: string;
-  simhash?: string;
-  uniqueTokenRatio?: number;
-  soft404Score?: number;
-  soft404Signals?: string[];
-  crawlTrapFlag?: boolean;
-  crawlTrapRisk?: number;
-  trapType?: string;
-  securityError?: string;
-  retries?: number;
-  clusterId?: number;
-  bytesReceived?: number;
-  linkRole?: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
-}
-
-export interface GraphEdge {
-  source: string;
-  target: string;
-  weight: number;
-}
-
-export interface ClusterInfo {
-  id: number;
-  count: number;
-  primaryUrl: string;
-  risk: 'low' | 'medium' | 'high';
-  sharedPathPrefix?: string;
-}
-
-export interface CrawlStats {
-  pagesFetched: number;
-  pagesCached: number;
-  pagesSkipped: number;
-  totalFound: number;
-}
-
-export class Graph {
-  nodes: Map<string, GraphNode> = new Map();
-  // Using string "source|target" to ensure uniqueness efficiently. Mapping to weight.
-  edges: Map<string, number> = new Map();
-  limitReached: boolean = false;
-  sessionStats: CrawlStats = {
-    pagesFetched: 0,
-    pagesCached: 0,
-    pagesSkipped: 0,
-    totalFound: 0
-  };
-  trapClusters: { pattern: string; type: string; count: number }[] = [];
-  duplicateClusters: { id: string; type: 'exact' | 'near' | 'template_heavy'; size: number; representative: string; severity: 'low' | 'medium' | 'high' }[] = [];
-  contentClusters: ClusterInfo[] = [];
-
-  /**
-   * Adds a node to the graph if it doesn't exist.
-   * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
-   * Depth is only set on creation (BFS guarantees shortest path first).
-   */
-  addNode(url: string, depth: number, status: number = 0) {
-    const existing = this.nodes.get(url);
-    if (!existing) {
-      this.nodes.set(url, {
-        url,
-        depth,
-        status,
-        inLinks: 0,
-        outLinks: 0
-      });
-    } else {
-      // Update status if we have a real one now (e.g. was 0/pending, now crawled)
-      if (status !== 0) {
-        existing.status = status;
-      }
-    }
-  }
-
-  updateNodeData(url: string, data: Partial<GraphNode>) {
-    const existing = this.nodes.get(url);
-    if (existing) {
-      Object.assign(existing, data);
-    }
-  }
-
-  /**
-   * Adds a directed edge between two nodes.
-   * Both nodes must exist in the graph.
-   * Updates inLinks and outLinks counts.
-   */
-  addEdge(source: string, target: string, weight: number = 1.0) {
-    const sourceNode = this.nodes.get(source);
-    const targetNode = this.nodes.get(target);
-
-    if (sourceNode && targetNode) {
-      const edgeKey = `${source}|${target}`;
-      if (!this.edges.has(edgeKey)) {
-        this.edges.set(edgeKey, weight);
-        sourceNode.outLinks++;
-        targetNode.inLinks++;
-      } else {
-        // If edge exists, keep highest weight (or could sum, but usually we just want the 'best' relationship)
-        const currentWeight = this.edges.get(edgeKey) || 0;
-        if (weight > currentWeight) {
-          this.edges.set(edgeKey, weight);
-        }
-      }
-    }
-  }
-
-  getNodes(): GraphNode[] {
-    return Array.from(this.nodes.values());
-  }
-
-  getEdges(): GraphEdge[] {
-    return Array.from(this.edges.entries()).map(([edge, weight]) => {
-      const [source, target] = edge.split('|');
-      return { source, target, weight };
-    });
-  }
-
-  toJSON() {
-    return {
-      nodes: this.getNodes(),
-      edges: this.getEdges(),
-      duplicateClusters: this.duplicateClusters,
-      contentClusters: this.contentClusters
-    };
-  }
-
-  static fromJSON(json: any): Graph {
-    const graph = new Graph();
-    if (json.nodes) {
-      for (const node of json.nodes) {
-        graph.nodes.set(node.url, { ...node });
-      }
-    }
-    if (json.edges) {
-      for (const edge of json.edges) {
-        const key = `${edge.source}|${edge.target}`;
-        graph.edges.set(key, edge.weight || 1.0);
-      }
-    }
-    if (json.duplicateClusters) {
-      graph.duplicateClusters = json.duplicateClusters;
-    }
-    if (json.contentClusters) {
-      graph.contentClusters = json.contentClusters;
-    }
-    return graph;
-  }
-}