@crawlith/core 0.1.1 → 0.1.2

package/src/graph/graph.ts

@@ -1,192 +0,0 @@
-export interface GraphNode {
-  url: string;
-  depth: number;
-  inLinks: number;
-  outLinks: number;
-  status: number;
-  canonical?: string;
-  noindex?: boolean;
-  nofollow?: boolean;
-  brokenLinks?: string[];
-  redirectChain?: string[];
-  incrementalStatus?: 'new' | 'changed' | 'unchanged' | 'deleted';
-  etag?: string;
-  lastModified?: string;
-  contentHash?: string;
-  html?: string;
-  pageRank?: number;
-  pageRankScore?: number;
-  authorityScore?: number;
-  hubScore?: number;
-  duplicateClusterId?: string;
-  duplicateType?: 'exact' | 'near' | 'template_heavy' | 'none';
-  isClusterPrimary?: boolean;
-  isCollapsed?: boolean;
-  collapseInto?: string;
-  simhash?: string;
-  uniqueTokenRatio?: number;
-  soft404Score?: number;
-  soft404Signals?: string[];
-  crawlTrapFlag?: boolean;
-  crawlTrapRisk?: number;
-  trapType?: string;
-  securityError?: string;
-  retries?: number;
-  clusterId?: number;
-  bytesReceived?: number;
-  linkRole?: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
-  crawlStatus?: string;
-  wordCount?: number;
-  thinContentScore?: number;
-  externalLinkRatio?: number;
-  orphanScore?: number;
-}
-
-export interface GraphEdge {
-  source: string;
-  target: string;
-  weight: number;
-}
-
-export interface ClusterInfo {
-  id: number;
-  count: number;
-  primaryUrl: string;
-  risk: 'low' | 'medium' | 'high';
-  sharedPathPrefix?: string;
-}
-
-export interface CrawlStats {
-  pagesFetched: number;
-  pagesCached: number;
-  pagesSkipped: number;
-  totalFound: number;
-}
-
-export class Graph {
-  nodes: Map<string, GraphNode> = new Map();
-  // Using JSON string of [source, target] to ensure uniqueness. Mapping to weight.
-  edges: Map<string, number> = new Map();
-  limitReached: boolean = false;
-  sessionStats: CrawlStats = {
-    pagesFetched: 0,
-    pagesCached: 0,
-    pagesSkipped: 0,
-    totalFound: 0
-  };
-  trapClusters: { pattern: string; type: string; count: number }[] = [];
-  duplicateClusters: { id: string; type: 'exact' | 'near' | 'template_heavy'; size: number; representative: string; severity: 'low' | 'medium' | 'high' }[] = [];
-  contentClusters: ClusterInfo[] = [];
-
-  /**
-   * Generates a unique key for an edge.
-   */
-  static getEdgeKey(source: string, target: string): string {
-    return JSON.stringify([source, target]);
-  }
-
-  /**
-   * Parses an edge key back into source and target.
-   */
-  static parseEdgeKey(key: string): { source: string; target: string } {
-    const [source, target] = JSON.parse(key);
-    return { source, target };
-  }
-
-  /**
-   * Adds a node to the graph if it doesn't exist.
-   * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
-   * Depth is only set on creation (BFS guarantees shortest path first).
-   */
-  addNode(url: string, depth: number, status: number = 0) {
-    const existing = this.nodes.get(url);
-    if (!existing) {
-      this.nodes.set(url, {
-        url,
-        depth,
-        status,
-        inLinks: 0,
-        outLinks: 0
-      });
-    } else {
-      // Update status if we have a real one now (e.g. was 0/pending, now crawled)
-      if (status !== 0) {
-        existing.status = status;
-      }
-    }
-  }
-
-  updateNodeData(url: string, data: Partial<GraphNode>) {
-    const existing = this.nodes.get(url);
-    if (existing) {
-      Object.assign(existing, data);
-    }
-  }
-
-  /**
-   * Adds a directed edge between two nodes.
-   * Both nodes must exist in the graph.
-   * Updates inLinks and outLinks counts.
-   */
-  addEdge(source: string, target: string, weight: number = 1.0) {
-    const sourceNode = this.nodes.get(source);
-    const targetNode = this.nodes.get(target);
-
-    if (sourceNode && targetNode) {
-      const edgeKey = Graph.getEdgeKey(source, target);
-      if (!this.edges.has(edgeKey)) {
-        this.edges.set(edgeKey, weight);
-        sourceNode.outLinks++;
-        targetNode.inLinks++;
-      } else {
-        // If edge exists, keep highest weight (or could sum, but usually we just want the 'best' relationship)
-        const currentWeight = this.edges.get(edgeKey) || 0;
-        if (weight > currentWeight) {
-          this.edges.set(edgeKey, weight);
-        }
-      }
-    }
-  }
-
-  getNodes(): GraphNode[] {
-    return Array.from(this.nodes.values());
-  }
-
-  getEdges(): GraphEdge[] {
-    return Array.from(this.edges.entries()).map(([edge, weight]) => {
-      const { source, target } = Graph.parseEdgeKey(edge);
-      return { source, target, weight };
-    });
-  }
-
-  toJSON() {
-    return {
-      nodes: this.getNodes(),
-      edges: this.getEdges(),
-      duplicateClusters: this.duplicateClusters,
-      contentClusters: this.contentClusters
-    };
-  }
-
-  static fromJSON(json: any): Graph {
-    const graph = new Graph();
-    if (json.nodes) {
-      for (const node of json.nodes) {
-        graph.nodes.set(node.url, { ...node });
-      }
-    }
-    if (json.edges) {
-      for (const edge of json.edges) {
-        const key = Graph.getEdgeKey(edge.source, edge.target);
-        graph.edges.set(key, edge.weight || 1.0);
-      }
-    }
-    if (json.duplicateClusters) {
-      graph.duplicateClusters = json.duplicateClusters;
-    }
-    if (json.contentClusters) {
-      graph.contentClusters = json.contentClusters;
-    }
-    return graph;
-  }
-}

package/src/graph/metrics.ts

@@ -1,125 +0,0 @@
-import { Graph, GraphNode } from './graph.js';
-
-export interface Metrics {
-  totalPages: number;
-  totalEdges: number;
-  orphanPages: string[];
-  nearOrphans: string[];
-  deepPages: string[];
-  topAuthorityPages: { url: string; authority: number }[];
-  averageOutDegree: number;
-  maxDepthFound: number;
-  crawlEfficiencyScore: number;
-  averageDepth: number;
-  structuralEntropy: number;
-  topPageRankPages: { url: string; score: number }[];
-  limitReached: boolean;
-  sessionStats?: {
-    pagesFetched: number;
-    pagesCached: number;
-    pagesSkipped: number;
-    totalFound: number;
-  };
-}
-
-export function calculateMetrics(graph: Graph, _maxDepth: number): Metrics {
-  const nodes = graph.getNodes();
-  const edges = graph.getEdges();
-
-  const totalPages = nodes.length;
-  const totalEdges = edges.length;
-
-  // Identify broken nodes
-  const brokenNodes = new Set(nodes.filter(n => n.status >= 400 || n.status === 0).map(n => n.url));
-
-  // Populate brokenLinks per node
-  for (const node of nodes) {
-    const nodeEdges = edges.filter(e => e.source === node.url);
-    const broken = nodeEdges
-      .map(e => e.target)
-      .filter(targetUrl => brokenNodes.has(targetUrl));
-
-    if (broken.length > 0) {
-      node.brokenLinks = broken;
-    }
-  }
-
-  // Authority Score (per node)
-  const maxInLinks = nodes.reduce((max, n) => Math.max(max, n.inLinks), 0);
-  const getAuthority = (node: GraphNode) => {
-    if (maxInLinks === 0) return 0;
-    return Math.log(1 + node.inLinks) / Math.log(1 + maxInLinks);
-  };
-
-  // orphanPages: inLinks === 0 && depth > 0
-  const orphanPages = nodes
-    .filter(n => n.inLinks === 0 && n.depth > 0)
-    .map(n => n.url);
-
-  // nearOrphans: inLinks === 1 && depth >= 3
-  const nearOrphans = nodes
-    .filter(n => n.inLinks === 1 && n.depth >= 3)
-    .map(n => n.url);
-
-  // deepPages: depth >= 4
-  const deepPages = nodes
-    .filter(n => n.depth >= 4) // Per requirement
-    .map(n => n.url);
-
-  // crawlEfficiencyScore: 1 - (deepPagesCount / totalPages)
-  const deepPagesCount = deepPages.length;
-  const crawlEfficiencyScore = totalPages > 0 ? 1 - (deepPagesCount / totalPages) : 1;
-
-  // averageDepth: sum(depth) / totalPages
-  const sumDepth = nodes.reduce((acc, n) => acc + n.depth, 0);
-  const averageDepth = totalPages > 0 ? sumDepth / totalPages : 0;
-
-  // structuralEntropy: Shannon entropy over outDegree distribution
-  const outDegreeCounts = new Map<number, number>();
-  nodes.forEach(n => {
-    outDegreeCounts.set(n.outLinks, (outDegreeCounts.get(n.outLinks) || 0) + 1);
-  });
-
-  let structuralEntropy = 0;
-  if (totalPages > 0) {
-    for (const count of outDegreeCounts.values()) {
-      const p = count / totalPages;
-      if (p > 0) {
-        structuralEntropy -= p * Math.log2(p);
-      }
-    }
-  }
-
-  // topAuthorityPages: Top 10 by authority
-  const topAuthorityPages = [...nodes]
-    .map(n => ({ url: n.url, authority: n.authorityScore ?? getAuthority(n) }))
-    .sort((a, b) => b.authority - a.authority)
-    .slice(0, 10);
-
-  // topPageRankPages: Top 10 by raw PageRank
-  const topPageRankPages = [...nodes]
-    .filter(n => n.pageRank !== undefined)
-    .map(n => ({ url: n.url, score: n.pageRank! }))
-    .sort((a, b) => b.score - a.score)
-    .slice(0, 10);
-
-  const averageOutDegree = totalPages > 0 ? totalEdges / totalPages : 0;
-  const maxDepthFound = nodes.reduce((max, n) => Math.max(max, n.depth), 0);
-
-  return {
-    totalPages,
-    totalEdges,
-    orphanPages,
-    nearOrphans,
-    deepPages,
-    topAuthorityPages,
-    averageOutDegree,
-    maxDepthFound,
-    crawlEfficiencyScore,
-    averageDepth,
-    structuralEntropy,
-    topPageRankPages,
-    limitReached: graph.limitReached,
-    sessionStats: graph.sessionStats
-  };
-}

package/src/graph/pagerank.ts

@@ -1,126 +0,0 @@
-import { Graph, GraphNode } from './graph.js';
-
-interface PageRankOptions {
-    dampingFactor?: number;
-    maxIterations?: number;
-    convergenceThreshold?: number;
-    soft404WeightThreshold?: number;
-}
-
-/**
- * Production-Grade Weighted PageRank Engine
- */
-export function computePageRank(graph: Graph, options: PageRankOptions = {}) {
-    const d = options.dampingFactor ?? 0.85;
-    const maxIterations = options.maxIterations ?? 40;
-    const epsilon = options.convergenceThreshold ?? 1e-5;
-    const soft404Threshold = options.soft404WeightThreshold ?? 0.8;
-
-    const allNodes = graph.getNodes();
-    const allEdges = graph.getEdges();
-
-    // 1. Filter Eligible Nodes
-    const eligibleNodes = allNodes.filter(node => {
-        if (node.noindex) return false;
-        if (node.isCollapsed) return false;
-        if (node.soft404Score && node.soft404Score > soft404Threshold) return false;
-        if (node.canonical && node.canonical !== node.url) return false;
-        if (node.status >= 400) return false; // Don't pass rank to broken pages
-        if (node.status === 0) return false; // Don't pass rank to uncrawled/external pages
-        return true;
-    });
-
-    const nodeCount = eligibleNodes.length;
-    if (nodeCount === 0) return;
-
-    const nodeUrls = eligibleNodes.map(n => n.url);
-    const nodeMap = new Map<string, GraphNode>();
-    eligibleNodes.forEach(n => nodeMap.set(n.url, n));
-
-    // Initialize PageRank
-    let pr = new Map<string, number>();
-    nodeUrls.forEach(url => pr.set(url, 1 / nodeCount));
-
-    // Pre-calculate weighted outbound sums and inverted adjacency
-    const outWeights = new Map<string, number>();
-    const incoming = new Map<string, { source: string; weight: number }[]>();
-    const sinks: string[] = [];
-
-    // Initialize outWeights for all eligible nodes
-    nodeUrls.forEach(url => outWeights.set(url, 0));
-
-    for (const edge of allEdges) {
-        if (nodeMap.has(edge.source) && nodeMap.has(edge.target)) {
-            const weight = edge.weight || 1.0;
-
-            const sources = incoming.get(edge.target) ?? [];
-            sources.push({ source: edge.source, weight });
-            incoming.set(edge.target, sources);
-
-            outWeights.set(edge.source, (outWeights.get(edge.source) || 0) + weight);
-        }
-    }
-
-    // Identify sinks
-    nodeUrls.forEach(url => {
-        if ((outWeights.get(url) || 0) === 0) {
-            sinks.push(url);
-        }
-    });
-
-    // Iterative Calculation
-    for (let i = 0; i < maxIterations; i++) {
-        const nextPr = new Map<string, number>();
-
-        // Calculate total rank from sinks to redistribute
-        let sinkRankTotal = 0;
-        for (const url of sinks) {
-            sinkRankTotal += pr.get(url) || 0;
-        }
-
-        const baseRank = (1 - d) / nodeCount + (d * sinkRankTotal / nodeCount);
-
-        for (const url of nodeUrls) {
-            let rankFromLinks = 0;
-            const sources = incoming.get(url) || [];
-
-            for (const edge of sources) {
-                const sourceRank = pr.get(edge.source) || 0;
-                const sourceOutWeight = outWeights.get(edge.source) || 1.0;
-                rankFromLinks += sourceRank * (edge.weight / sourceOutWeight);
-            }
-
-            const newRank = baseRank + d * rankFromLinks;
-            nextPr.set(url, newRank);
-        }
-
-        // Convergence check
-        let maxDelta = 0;
-        for (const url of nodeUrls) {
-            const delta = Math.abs(nextPr.get(url)! - pr.get(url)!);
-            if (delta > maxDelta) maxDelta = delta;
-        }
-
-        pr = nextPr;
-
-        if (maxDelta < epsilon) break;
-    }
-
-    // 2. Normalization (0-100)
-    const ranks = Array.from(pr.values());
-    const minPR = Math.min(...ranks);
-    const maxPR = Math.max(...ranks);
-    const range = maxPR - minPR;
-
-    for (const node of eligibleNodes) {
-        const rawRank = pr.get(node.url)!;
-        node.pageRank = rawRank;
-
-        if (range > 1e-12) {
-            node.pageRankScore = 100 * (rawRank - minPR) / range;
-        } else {
-            // If there's no range, all eligible pages are equally important.
-            node.pageRankScore = 100;
-        }
-    }
-}

package/src/graph/simhash.ts

@@ -1,76 +0,0 @@
-export class SimHash {
-    private static FNV_PRIME = 1099511628211n;
-    private static FNV_OFFSET_BASIS = 14695981039346656037n;
-    private static MAX_UINT64 = 0xffffffffffffffffn;
-    public static readonly BANDS = 4;
-    public static readonly BAND_WIDTH = 16;
-
-    /**
-     * Generates a 64-bit FNV-1a hash for a given string token.
-     */
-    static fnv1a64(token: string): bigint {
-        let hash = this.FNV_OFFSET_BASIS;
-        const len = token.length;
-        for (let i = 0; i < len; i++) {
-            hash ^= BigInt(token.charCodeAt(i));
-            // BigInt safe multiplication modulo 2^64
-            hash = (hash * this.FNV_PRIME) & this.MAX_UINT64;
-        }
-        return hash;
-    }
-
-    /**
-     * Generates a 64-bit SimHash from an array of tokens.
-     */
-    static generate(tokens: string[]): bigint {
-        const v = new Int32Array(64);
-
-        for (const token of tokens) {
-            const hash = this.fnv1a64(token);
-            for (let i = 0n; i < 64n; i++) {
-                const bit = (hash >> i) & 1n;
-                if (bit === 1n) {
-                    v[Number(i)]++;
-                } else {
-                    v[Number(i)]--;
-                }
-            }
-        }
-
-        let simhash = 0n;
-        for (let i = 0n; i < 64n; i++) {
-            if (v[Number(i)] > 0) {
-                simhash |= (1n << i);
-            }
-        }
-
-        return simhash;
-    }
-
-    /**
-     * Splits a 64-bit SimHash into 4 bands of 16 bits.
-     */
-    static getBands(simhash: bigint): number[] {
-        const bands: number[] = [];
-        for (let i = 0; i < SimHash.BANDS; i++) {
-            // Extract 16-bit chunks
-            const chunk = Number((simhash >> BigInt(i * SimHash.BAND_WIDTH)) & 0xFFFFn);
-            bands.push(chunk);
-        }
-        return bands;
-    }
-
-    /**
-     * Computes the Hamming distance between two 64-bit hashes.
-     */
-    static hammingDistance(a: bigint, b: bigint): number {
-        let xor = a ^ b;
-        let distance = 0;
-        while (xor > 0n) {
-            // Kernighan's bit counting
-            xor &= xor - 1n;
-            distance++;
-        }
-        return distance;
-    }
-}

package/src/index.ts

@@ -1,33 +0,0 @@
-export * from './crawler/crawl.js';
-export * from './crawler/normalize.js';
-export * from './crawler/metricsRunner.js';
-export * from './graph/metrics.js';
-export * from './report/html.js';
-export * from './report/crawl_template.js';
-export * from './report/crawlExport.js';
-export * from './graph/graph.js';
-export * from './diff/compare.js';
-export * from './scoring/orphanSeverity.js';
-export * from './graph/pagerank.js';
-export * from './graph/duplicate.js';
-export * from './graph/cluster.js';
-export * from './scoring/health.js';
-export * from './scoring/hits.js';
-export * from './analysis/analyze.js';
-export * from './analysis/content.js';
-export * from './analysis/seo.js';
-export * from './analysis/images.js';
-export * from './analysis/links.js';
-export * from './audit/index.js';
-export * from './audit/types.js';
-export * from './db/index.js';
-export * from './db/graphLoader.js';
-export * from './db/repositories/SiteRepository.js';
-export * from './db/repositories/SnapshotRepository.js';
-export * from './db/repositories/PageRepository.js';
-export * from './db/repositories/EdgeRepository.js';
-export * from './db/repositories/MetricsRepository.js';
-export * from './lock/lockManager.js';
-export * from './lock/hashKey.js';
-export * from './utils/version.js';
-export * from './events.js';

package/src/lock/hashKey.ts

@@ -1,51 +0,0 @@
-import crypto from 'node:crypto';
-import { normalizeUrl } from '../crawler/normalize.js';
-
-// Flags that change the nature of the crawl and should be part of the lock key
-const RELEVANT_FLAGS = [
-  'limit',
-  'depth',
-  'output',
-  'sitemap',
-  'incremental',
-  'detectSoft404',
-  'detectTraps',
-  'includeSubdomains',
-  'allow',
-  'deny',
-  'proxy',
-  'ua',
-  'maxRedirects',
-  'rate',
-  'maxBytes',
-  'concurrency'
-];
-
-export function generateLockKey(commandName: string, targetUrl: string, options: any): string {
-  // Respect the query stripping option consistent with crawl logic
-  const stripQuery = !options.query;
-
-  const normalizedTarget = normalizeUrl(targetUrl, '', { stripQuery }) || targetUrl;
-
-  // Extract relevant options in a deterministic order
-  const lockOptions: Record<string, any> = {};
-  for (const key of RELEVANT_FLAGS) {
-    if (options[key] !== undefined) {
-      lockOptions[key] = options[key];
-    }
-  }
-
-  // Create composite key object
-  const compositeKey = {
-    command: commandName,
-    target: normalizedTarget,
-    options: lockOptions
-  };
-
-  // Stringify and hash
-  // Since we inserted keys in a deterministic order (RELEVANT_FLAGS order),
-  // JSON.stringify will produce a stable string in V8/Node.js.
-  const stableString = JSON.stringify(compositeKey);
-
-  return crypto.createHash('sha256').update(stableString).digest('hex');
-}