@crawlith/core 0.1.0

package/src/graph/duplicate.ts

@@ -0,0 +1,286 @@
+import { Graph, GraphNode } from './graph.js';
+import { SimHash } from './simhash.js';
+
+export interface DuplicateOptions {
+    collapse?: boolean;
+    simhashThreshold?: number; // Hamming distance threshold (default: 3)
+}
+
+interface DuplicateCluster {
+    id: string;
+    type: 'exact' | 'near' | 'template_heavy';
+    nodes: GraphNode[];
+    representative?: string;
+    severity?: 'low' | 'medium' | 'high';
+}
+
+/**
+ * Detects exact and near duplicates, identifies canonical conflicts,
+ * and performs non-destructive collapse of edges.
+ */
+export function detectDuplicates(graph: Graph, options: DuplicateOptions = {}) {
+    const collapse = options.collapse !== false; // Default to true
+    const threshold = options.simhashThreshold ?? 3;
+
+    const exactClusters: DuplicateCluster[] = [];
+    const nearClusters: DuplicateCluster[] = [];
+
+    const nodes = graph.getNodes();
+
+    // Phase 1 & 2: Exact Duplicate Detection
+    const exactMap = new Map<string, GraphNode[]>();
+    for (const node of nodes) {
+        if (!node.contentHash || node.status !== 200) continue;
+
+        // Safety check: if there's no soft404 signal (soft404 is handled elsewhere, but just filter 200 OKs)
+        let arr = exactMap.get(node.contentHash);
+        if (!arr) {
+            arr = [];
+            exactMap.set(node.contentHash, arr);
+        }
+        arr.push(node);
+    }
+
+    // Nodes that are NOT part of an exact duplicate group are candidates for near duplicate checks
+    const nearCandidates: GraphNode[] = [];
+    let clusterCounter = 1;
+
+    for (const [_hash, group] of exactMap.entries()) {
+        if (group.length > 1) {
+            const id = `cluster_exact_${clusterCounter++}`;
+            exactClusters.push({ id, type: 'exact', nodes: group });
+            // Mark nodes
+            for (const n of group) {
+                n.duplicateClusterId = id;
+                n.duplicateType = 'exact';
+            }
+        } else {
+            nearCandidates.push(group[0]);
+        }
+    }
+
+    // Phase 3: Near Duplicate Detection (SimHash with Bands)
+    // 64-bit simhash -> split into 4 bands of 16 bits.
+    const bandsMaps = [
+        new Map<number, GraphNode[]>(),
+        new Map<number, GraphNode[]>(),
+        new Map<number, GraphNode[]>(),
+        new Map<number, GraphNode[]>()
+    ];
+
+    for (const node of nearCandidates) {
+        if (!node.simhash) continue;
+        const simhash = BigInt(node.simhash);
+
+        // Extract 16 bit bands
+        const b0 = Number(simhash & 0xFFFFn);
+        const b1 = Number((simhash >> 16n) & 0xFFFFn);
+        const b2 = Number((simhash >> 32n) & 0xFFFFn);
+        const b3 = Number((simhash >> 48n) & 0xFFFFn);
+
+        const bands = [b0, b1, b2, b3];
+        for (let i = 0; i < 4; i++) {
+            let arr = bandsMaps[i].get(bands[i]);
+            if (!arr) {
+                arr = [];
+                bandsMaps[i].set(bands[i], arr);
+            }
+            arr.push(node);
+        }
+    }
+
+    // Find candidate pairs
+    const nearGroupMap = new Map<string, Set<GraphNode>>(); // node.url -> cluster set
+    const checkedPairs = new Set<string>();
+
+    for (let i = 0; i < 4; i++) {
+        for (const [_bandVal, bucketNodes] of bandsMaps[i].entries()) {
+            if (bucketNodes.length < 2) continue; // nothing to compare
+
+            // Compare all nodes in this bucket
+            for (let j = 0; j < bucketNodes.length; j++) {
+                for (let k = j + 1; k < bucketNodes.length; k++) {
+                    const n1 = bucketNodes[j];
+                    const n2 = bucketNodes[k];
+
+                    // Ensure n1 < n2 lexicographically to avoid duplicate pairs
+                    const [a, b] = n1.url < n2.url ? [n1, n2] : [n2, n1];
+                    const pairKey = `${a.url}|${b.url}`;
+
+                    if (checkedPairs.has(pairKey)) continue;
+                    checkedPairs.add(pairKey);
+
+                    const dist = SimHash.hammingDistance(BigInt(a.simhash!), BigInt(b.simhash!));
+                    if (dist <= threshold) {
+                        // They are near duplicates. 
+                        // Find or create their cluster set using union-find or reference propagation
+                        const setA = nearGroupMap.get(a.url);
+                        const setB = nearGroupMap.get(b.url);
+
+                        if (!setA && !setB) {
+                            const newSet = new Set<GraphNode>([a, b]);
+                            nearGroupMap.set(a.url, newSet);
+                            nearGroupMap.set(b.url, newSet);
+                        } else if (setA && !setB) {
+                            setA.add(b);
+                            nearGroupMap.set(b.url, setA);
+                        } else if (setB && !setA) {
+                            setB.add(a);
+                            nearGroupMap.set(a.url, setB);
+                        } else if (setA && setB && setA !== setB) {
+                            // Merge sets
+                            for (const node of setB) {
+                                setA.add(node);
+                                nearGroupMap.set(node.url, setA);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // Compile near duplicate clusters (deduplicated by Set reference)
+    const uniqueNearSets = new Set<Set<GraphNode>>();
+    for (const group of nearGroupMap.values()) {
+        uniqueNearSets.add(group);
+    }
+
+    for (const groupSet of uniqueNearSets) {
+        if (groupSet.size > 1) {
+            const id = `cluster_near_${clusterCounter++}`;
+            const groupArr = Array.from(groupSet);
+            nearClusters.push({ id, type: 'near', nodes: groupArr });
+            for (const n of groupArr) {
+                n.duplicateClusterId = id;
+                n.duplicateType = 'near';
+            }
+        }
+    }
+
+    const allClusters = [...exactClusters, ...nearClusters];
+
+    // Phase 4: Template-Heavy Detection
+    // Mark classes as 'template_heavy' if ratio < 0.3
+    for (const cluster of allClusters) {
+        const avgRatio = cluster.nodes.reduce((sum, n) => sum + (n.uniqueTokenRatio || 0), 0) / cluster.nodes.length;
+        if (avgRatio < 0.3) {
+            cluster.type = 'template_heavy';
+            cluster.nodes.forEach(n => n.duplicateType = 'template_heavy');
+        }
+    }
+
+    // Phase 5: Canonical Conflict & Representative Selection
+    for (const cluster of allClusters) {
+        const canonicals = new Set<string>();
+        let hasMissing = false;
+
+        for (const n of cluster.nodes) {
+            if (!n.canonical) hasMissing = true;
+            // We compare full absolute canonical URLs (assuming they are normalized during crawl)
+            else canonicals.add(n.canonical);
+        }
+
+        if (hasMissing || canonicals.size > 1) {
+            cluster.severity = 'high';
+        } else if (cluster.type === 'near') {
+            cluster.severity = 'medium';
+        } else {
+            cluster.severity = 'low';
+        }
+
+        // Phase 6: Select Representative
+        // 1. Valid Canonical target in cluster
+        // 2. Highest internal in-degree
+        // 3. Shortest URL
+        // 4. First discovered (relying on array order, which is from BFS map roughly)
+        let representativeNode = cluster.nodes[0];
+
+        // Evaluate best rep
+        const urlsInCluster = new Set(cluster.nodes.map(n => n.url));
+        const validCanonicals = cluster.nodes.filter(n => n.canonical && urlsInCluster.has(n.canonical) && n.url === n.canonical);
+
+        if (validCanonicals.length > 0) {
+            representativeNode = validCanonicals[0]; // If multiple, just pick first matching self
+        } else {
+            representativeNode = cluster.nodes.reduce((best, current) => {
+                if (current.inLinks > best.inLinks) return current;
+                if (current.inLinks < best.inLinks) return best;
+                if (current.url.length < best.url.length) return current;
+                return best;
+            });
+        }
+
+        cluster.representative = representativeNode.url;
+
+        cluster.nodes.forEach(n => {
+            n.isClusterPrimary = n.url === representativeNode.url;
+            n.isCollapsed = false; // default for JSON
+            n.collapseInto = undefined;
+        });
+
+        // Push to Graph's final cluster list
+        graph.duplicateClusters.push({
+            id: cluster.id,
+            type: cluster.type,
+            size: cluster.nodes.length,
+            representative: representativeNode.url,
+            severity: cluster.severity!
+        });
+
+        // Controlled Collapse
+        if (collapse) {
+            for (const n of cluster.nodes) {
+                if (n.url !== representativeNode.url) {
+                    n.isCollapsed = true;
+                    n.collapseInto = representativeNode.url;
+                }
+            }
+        }
+    }
+
+    // Final Edge Transfer if Collapsing
+    if (collapse) {
+        const edges = graph.getEdges();
+        const updatedEdges = new Map<string, number>();
+
+        for (const edge of edges) {
+            const sourceNode = graph.nodes.get(edge.source);
+            const targetNode = graph.nodes.get(edge.target);
+
+            if (!sourceNode || !targetNode) continue;
+
+            // We do NOT modify source structure for out-bound edges of collapsed nodes? 
+            // Spec: "Ignore edges from collapsed nodes. Transfer inbound edges to representative."
+            // Actually, if a node links TO a collapsed node, we repoint the edge to the representative.
+            // If a collapsed node links to X, we ignore it (PageRank will filter it out).
+
+            const actualSource = edge.source;
+            // repoint target
+            const actualTarget = targetNode.isCollapsed && targetNode.collapseInto ? targetNode.collapseInto : edge.target;
+
+            // Skip self-referential edges caused by repointing
+            if (actualSource === actualTarget) continue;
+
+            const edgeKey = `${actualSource}|${actualTarget}`;
+            const existingWeight = updatedEdges.get(edgeKey) || 0;
+            updatedEdges.set(edgeKey, Math.max(existingWeight, edge.weight)); // deduplicate
+        }
+
+        // Update graph edges in-place
+        graph.edges = updatedEdges;
+
+        // Re-calculate inLinks and outLinks based on collapsed edges
+        for (const node of graph.getNodes()) {
+            node.inLinks = 0;
+            node.outLinks = 0;
+        }
+        for (const [edgeKey, _weight] of updatedEdges.entries()) {
+            const [src, tgt] = edgeKey.split('|');
+            const sn = graph.nodes.get(src);
+            const tn = graph.nodes.get(tgt);
+            if (sn) sn.outLinks++;
+            if (tn) tn.inLinks++;
+        }
+    }
+}

package/src/graph/graph.ts

@@ -0,0 +1,172 @@
+export interface GraphNode {
+  url: string;
+  depth: number;
+  inLinks: number;
+  outLinks: number;
+  status: number;
+  canonical?: string;
+  noindex?: boolean;
+  nofollow?: boolean;
+  brokenLinks?: string[];
+  redirectChain?: string[];
+  incrementalStatus?: 'new' | 'changed' | 'unchanged' | 'deleted';
+  etag?: string;
+  lastModified?: string;
+  contentHash?: string;
+  html?: string;
+  pageRank?: number;
+  pageRankScore?: number;
+  authorityScore?: number;
+  hubScore?: number;
+  duplicateClusterId?: string;
+  duplicateType?: 'exact' | 'near' | 'template_heavy' | 'none';
+  isClusterPrimary?: boolean;
+  isCollapsed?: boolean;
+  collapseInto?: string;
+  simhash?: string;
+  uniqueTokenRatio?: number;
+  soft404Score?: number;
+  soft404Signals?: string[];
+  crawlTrapFlag?: boolean;
+  crawlTrapRisk?: number;
+  trapType?: string;
+  securityError?: string;
+  retries?: number;
+  clusterId?: number;
+  bytesReceived?: number;
+  linkRole?: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
+}
+
+export interface GraphEdge {
+  source: string;
+  target: string;
+  weight: number;
+}
+
+export interface ClusterInfo {
+  id: number;
+  count: number;
+  primaryUrl: string;
+  risk: 'low' | 'medium' | 'high';
+  sharedPathPrefix?: string;
+}
+
+export interface CrawlStats {
+  pagesFetched: number;
+  pagesCached: number;
+  pagesSkipped: number;
+  totalFound: number;
+}
+
+export class Graph {
+  nodes: Map<string, GraphNode> = new Map();
+  // Using string "source|target" to ensure uniqueness efficiently. Mapping to weight.
+  edges: Map<string, number> = new Map();
+  limitReached: boolean = false;
+  sessionStats: CrawlStats = {
+    pagesFetched: 0,
+    pagesCached: 0,
+    pagesSkipped: 0,
+    totalFound: 0
+  };
+  trapClusters: { pattern: string; type: string; count: number }[] = [];
+  duplicateClusters: { id: string; type: 'exact' | 'near' | 'template_heavy'; size: number; representative: string; severity: 'low' | 'medium' | 'high' }[] = [];
+  contentClusters: ClusterInfo[] = [];
+
+  /**
+   * Adds a node to the graph if it doesn't exist.
+   * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
+   * Depth is only set on creation (BFS guarantees shortest path first).
+   */
+  addNode(url: string, depth: number, status: number = 0) {
+    const existing = this.nodes.get(url);
+    if (!existing) {
+      this.nodes.set(url, {
+        url,
+        depth,
+        status,
+        inLinks: 0,
+        outLinks: 0
+      });
+    } else {
+      // Update status if we have a real one now (e.g. was 0/pending, now crawled)
+      if (status !== 0) {
+        existing.status = status;
+      }
+    }
+  }
+
+  updateNodeData(url: string, data: Partial<GraphNode>) {
+    const existing = this.nodes.get(url);
+    if (existing) {
+      Object.assign(existing, data);
+    }
+  }
+
+  /**
+   * Adds a directed edge between two nodes.
+   * Both nodes must exist in the graph.
+   * Updates inLinks and outLinks counts.
+   */
+  addEdge(source: string, target: string, weight: number = 1.0) {
+    const sourceNode = this.nodes.get(source);
+    const targetNode = this.nodes.get(target);
+
+    if (sourceNode && targetNode) {
+      const edgeKey = `${source}|${target}`;
+      if (!this.edges.has(edgeKey)) {
+        this.edges.set(edgeKey, weight);
+        sourceNode.outLinks++;
+        targetNode.inLinks++;
+      } else {
+        // If edge exists, keep highest weight (or could sum, but usually we just want the 'best' relationship)
+        const currentWeight = this.edges.get(edgeKey) || 0;
+        if (weight > currentWeight) {
+          this.edges.set(edgeKey, weight);
+        }
+      }
+    }
+  }
+
+  getNodes(): GraphNode[] {
+    return Array.from(this.nodes.values());
+  }
+
+  getEdges(): GraphEdge[] {
+    return Array.from(this.edges.entries()).map(([edge, weight]) => {
+      const [source, target] = edge.split('|');
+      return { source, target, weight };
+    });
+  }
+
+  toJSON() {
+    return {
+      nodes: this.getNodes(),
+      edges: this.getEdges(),
+      duplicateClusters: this.duplicateClusters,
+      contentClusters: this.contentClusters
+    };
+  }
+
+  static fromJSON(json: any): Graph {
+    const graph = new Graph();
+    if (json.nodes) {
+      for (const node of json.nodes) {
+        graph.nodes.set(node.url, { ...node });
+      }
+    }
+    if (json.edges) {
+      for (const edge of json.edges) {
+        const key = `${edge.source}|${edge.target}`;
+        graph.edges.set(key, edge.weight || 1.0);
+      }
+    }
+    if (json.duplicateClusters) {
+      graph.duplicateClusters = json.duplicateClusters;
+    }
+    if (json.contentClusters) {
+      graph.contentClusters = json.contentClusters;
+    }
+    return graph;
+  }
+}

package/src/graph/metrics.ts

@@ -0,0 +1,110 @@
+import { Graph, GraphNode } from './graph.js';
+
+export interface Metrics {
+  totalPages: number;
+  totalEdges: number;
+  orphanPages: string[];
+  nearOrphans: string[];
+  deepPages: string[];
+  topAuthorityPages: { url: string; authority: number }[];
+  averageOutDegree: number;
+  maxDepthFound: number;
+  crawlEfficiencyScore: number;
+  averageDepth: number;
+  structuralEntropy: number;
+  topPageRankPages: { url: string; score: number }[];
+  limitReached: boolean;
+  sessionStats?: {
+    pagesFetched: number;
+    pagesCached: number;
+    pagesSkipped: number;
+    totalFound: number;
+  };
+}
+
+export function calculateMetrics(graph: Graph, _maxDepth: number): Metrics {
+  const nodes = graph.getNodes();
+  const edges = graph.getEdges();
+
+  const totalPages = nodes.length;
+  const totalEdges = edges.length;
+
+  // Authority Score (per node)
+  const maxInLinks = nodes.reduce((max, n) => Math.max(max, n.inLinks), 0);
+  const getAuthority = (node: GraphNode) => {
+    if (maxInLinks === 0) return 0;
+    return Math.log(1 + node.inLinks) / Math.log(1 + maxInLinks);
+  };
+
+  // orphanPages: inLinks === 0 && depth > 0
+  const orphanPages = nodes
+    .filter(n => n.inLinks === 0 && n.depth > 0)
+    .map(n => n.url);
+
+  // nearOrphans: inLinks === 1 && depth >= 3
+  const nearOrphans = nodes
+    .filter(n => n.inLinks === 1 && n.depth >= 3)
+    .map(n => n.url);
+
+  // deepPages: depth >= 4
+  const deepPages = nodes
+    .filter(n => n.depth >= 4) // Per requirement
+    .map(n => n.url);
+
+  // crawlEfficiencyScore: 1 - (deepPagesCount / totalPages)
+  const deepPagesCount = deepPages.length;
+  const crawlEfficiencyScore = totalPages > 0 ? 1 - (deepPagesCount / totalPages) : 1;
+
+  // averageDepth: sum(depth) / totalPages
+  const sumDepth = nodes.reduce((acc, n) => acc + n.depth, 0);
+  const averageDepth = totalPages > 0 ? sumDepth / totalPages : 0;
+
+  // structuralEntropy: Shannon entropy over outDegree distribution
+  const outDegreeCounts = new Map<number, number>();
+  nodes.forEach(n => {
+    outDegreeCounts.set(n.outLinks, (outDegreeCounts.get(n.outLinks) || 0) + 1);
+  });
+
+  let structuralEntropy = 0;
+  if (totalPages > 0) {
+    for (const count of outDegreeCounts.values()) {
+      const p = count / totalPages;
+      if (p > 0) {
+        structuralEntropy -= p * Math.log2(p);
+      }
+    }
+  }
+
+  // topAuthorityPages: Top 10 by authority
+  const topAuthorityPages = [...nodes]
+    .map(n => ({ url: n.url, authority: n.authorityScore ?? getAuthority(n) }))
+    .sort((a, b) => b.authority - a.authority)
+    .slice(0, 10);
+
+  // topPageRankPages: Top 10 by raw PageRank
+  const topPageRankPages = [...nodes]
+    .filter(n => n.pageRank !== undefined)
+    .map(n => ({ url: n.url, score: n.pageRank! }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, 10);
+
+  const averageOutDegree = totalPages > 0 ? totalEdges / totalPages : 0;
+  const maxDepthFound = nodes.reduce((max, n) => Math.max(max, n.depth), 0);
+
+  return {
+    totalPages,
+    totalEdges,
+    orphanPages,
+    nearOrphans,
+    deepPages,
+    topAuthorityPages,
+    averageOutDegree,
+    maxDepthFound,
+    crawlEfficiencyScore,
+    averageDepth,
+    structuralEntropy,
+    topPageRankPages,
+    limitReached: graph.limitReached,
+    sessionStats: graph.sessionStats
+  };
+}

package/src/graph/pagerank.ts

@@ -0,0 +1,125 @@
+import { Graph, GraphNode } from './graph.js';
+
+interface PageRankOptions {
+    dampingFactor?: number;
+    maxIterations?: number;
+    convergenceThreshold?: number;
+    soft404WeightThreshold?: number;
+}
+
+/**
+ * Production-Grade Weighted PageRank Engine
+ */
+export function computePageRank(graph: Graph, options: PageRankOptions = {}) {
+    const d = options.dampingFactor ?? 0.85;
+    const maxIterations = options.maxIterations ?? 40;
+    const epsilon = options.convergenceThreshold ?? 1e-5;
+    const soft404Threshold = options.soft404WeightThreshold ?? 0.8;
+
+    const allNodes = graph.getNodes();
+    const allEdges = graph.getEdges();
+
+    // 1. Filter Eligible Nodes
+    const eligibleNodes = allNodes.filter(node => {
+        if (node.noindex) return false;
+        if (node.isCollapsed) return false;
+        if (node.soft404Score && node.soft404Score > soft404Threshold) return false;
+        if (node.canonical && node.canonical !== node.url) return false;
+        if (node.status >= 400) return false; // Don't pass rank to broken pages
+        return true;
+    });
+
+    const nodeCount = eligibleNodes.length;
+    if (nodeCount === 0) return;
+
+    const nodeUrls = eligibleNodes.map(n => n.url);
+    const nodeMap = new Map<string, GraphNode>();
+    eligibleNodes.forEach(n => nodeMap.set(n.url, n));
+
+    // Initialize PageRank
+    let pr = new Map<string, number>();
+    nodeUrls.forEach(url => pr.set(url, 1 / nodeCount));
+
+    // Pre-calculate weighted outbound sums and inverted adjacency
+    const outWeights = new Map<string, number>();
+    const incoming = new Map<string, { source: string; weight: number }[]>();
+    const sinks: string[] = [];
+
+    // Initialize outWeights for all eligible nodes
+    nodeUrls.forEach(url => outWeights.set(url, 0));
+
+    for (const edge of allEdges) {
+        if (nodeMap.has(edge.source) && nodeMap.has(edge.target)) {
+            const weight = edge.weight || 1.0;
+
+            const sources = incoming.get(edge.target) ?? [];
+            sources.push({ source: edge.source, weight });
+            incoming.set(edge.target, sources);
+
+            outWeights.set(edge.source, (outWeights.get(edge.source) || 0) + weight);
+        }
+    }
+
+    // Identify sinks
+    nodeUrls.forEach(url => {
+        if ((outWeights.get(url) || 0) === 0) {
+            sinks.push(url);
+        }
+    });
+
+    // Iterative Calculation
+    for (let i = 0; i < maxIterations; i++) {
+        const nextPr = new Map<string, number>();
+
+        // Calculate total rank from sinks to redistribute
+        let sinkRankTotal = 0;
+        for (const url of sinks) {
+            sinkRankTotal += pr.get(url) || 0;
+        }
+
+        const baseRank = (1 - d) / nodeCount + (d * sinkRankTotal / nodeCount);
+
+        for (const url of nodeUrls) {
+            let rankFromLinks = 0;
+            const sources = incoming.get(url) || [];
+
+            for (const edge of sources) {
+                const sourceRank = pr.get(edge.source) || 0;
+                const sourceOutWeight = outWeights.get(edge.source) || 1.0;
+                rankFromLinks += sourceRank * (edge.weight / sourceOutWeight);
+            }
+
+            const newRank = baseRank + d * rankFromLinks;
+            nextPr.set(url, newRank);
+        }
+
+        // Convergence check
+        let maxDelta = 0;
+        for (const url of nodeUrls) {
+            const delta = Math.abs(nextPr.get(url)! - pr.get(url)!);
+            if (delta > maxDelta) maxDelta = delta;
+        }
+
+        pr = nextPr;
+
+        if (maxDelta < epsilon) break;
+    }
+
+    // 2. Normalization (0-100)
+    const ranks = Array.from(pr.values());
+    const minPR = Math.min(...ranks);
+    const maxPR = Math.max(...ranks);
+    const range = maxPR - minPR;
+
+    for (const node of eligibleNodes) {
+        const rawRank = pr.get(node.url)!;
+        node.pageRank = rawRank;
+
+        if (range > 1e-12) {
+            node.pageRankScore = 100 * (rawRank - minPR) / range;
+        } else {
+            // If there's no range, all eligible pages are equally important.
+            node.pageRankScore = 100;
+        }
+    }
+}