@crawlith/core 0.1.0 → 0.1.2

package/src/graph/metrics.ts

@@ -1,110 +0,0 @@
-import { Graph, GraphNode } from './graph.js';
-
-export interface Metrics {
-  totalPages: number;
-  totalEdges: number;
-  orphanPages: string[];
-  nearOrphans: string[];
-  deepPages: string[];
-  topAuthorityPages: { url: string; authority: number }[];
-  averageOutDegree: number;
-  maxDepthFound: number;
-  crawlEfficiencyScore: number;
-  averageDepth: number;
-  structuralEntropy: number;
-  topPageRankPages: { url: string; score: number }[];
-  limitReached: boolean;
-  sessionStats?: {
-    pagesFetched: number;
-    pagesCached: number;
-    pagesSkipped: number;
-    totalFound: number;
-  };
-}
-
-export function calculateMetrics(graph: Graph, _maxDepth: number): Metrics {
-  const nodes = graph.getNodes();
-  const edges = graph.getEdges();
-
-  const totalPages = nodes.length;
-  const totalEdges = edges.length;
-
-  // Authority Score (per node)
-  const maxInLinks = nodes.reduce((max, n) => Math.max(max, n.inLinks), 0);
-  const getAuthority = (node: GraphNode) => {
-    if (maxInLinks === 0) return 0;
-    return Math.log(1 + node.inLinks) / Math.log(1 + maxInLinks);
-  };
-
-  // orphanPages: inLinks === 0 && depth > 0
-  const orphanPages = nodes
-    .filter(n => n.inLinks === 0 && n.depth > 0)
-    .map(n => n.url);
-
-  // nearOrphans: inLinks === 1 && depth >= 3
-  const nearOrphans = nodes
-    .filter(n => n.inLinks === 1 && n.depth >= 3)
-    .map(n => n.url);
-
-  // deepPages: depth >= 4
-  const deepPages = nodes
-    .filter(n => n.depth >= 4) // Per requirement
-    .map(n => n.url);
-
-  // crawlEfficiencyScore: 1 - (deepPagesCount / totalPages)
-  const deepPagesCount = deepPages.length;
-  const crawlEfficiencyScore = totalPages > 0 ? 1 - (deepPagesCount / totalPages) : 1;
-
-  // averageDepth: sum(depth) / totalPages
-  const sumDepth = nodes.reduce((acc, n) => acc + n.depth, 0);
-  const averageDepth = totalPages > 0 ? sumDepth / totalPages : 0;
-
-  // structuralEntropy: Shannon entropy over outDegree distribution
-  const outDegreeCounts = new Map<number, number>();
-  nodes.forEach(n => {
-    outDegreeCounts.set(n.outLinks, (outDegreeCounts.get(n.outLinks) || 0) + 1);
-  });
-
-  let structuralEntropy = 0;
-  if (totalPages > 0) {
-    for (const count of outDegreeCounts.values()) {
-      const p = count / totalPages;
-      if (p > 0) {
-        structuralEntropy -= p * Math.log2(p);
-      }
-    }
-  }
-
-  // topAuthorityPages: Top 10 by authority
-  const topAuthorityPages = [...nodes]
-    .map(n => ({ url: n.url, authority: n.authorityScore ?? getAuthority(n) }))
-    .sort((a, b) => b.authority - a.authority)
-    .slice(0, 10);
-
-  // topPageRankPages: Top 10 by raw PageRank
-  const topPageRankPages = [...nodes]
-    .filter(n => n.pageRank !== undefined)
-    .map(n => ({ url: n.url, score: n.pageRank! }))
-    .sort((a, b) => b.score - a.score)
-    .slice(0, 10);
-
-  const averageOutDegree = totalPages > 0 ? totalEdges / totalPages : 0;
-  const maxDepthFound = nodes.reduce((max, n) => Math.max(max, n.depth), 0);
-
-  return {
-    totalPages,
-    totalEdges,
-    orphanPages,
-    nearOrphans,
-    deepPages,
-    topAuthorityPages,
-    averageOutDegree,
-    maxDepthFound,
-    crawlEfficiencyScore,
-    averageDepth,
-    structuralEntropy,
-    topPageRankPages,
-    limitReached: graph.limitReached,
-    sessionStats: graph.sessionStats
-  };
-}

package/src/graph/pagerank.ts

@@ -1,125 +0,0 @@
-import { Graph, GraphNode } from './graph.js';
-
-interface PageRankOptions {
-    dampingFactor?: number;
-    maxIterations?: number;
-    convergenceThreshold?: number;
-    soft404WeightThreshold?: number;
-}
-
-/**
- * Production-Grade Weighted PageRank Engine
- */
-export function computePageRank(graph: Graph, options: PageRankOptions = {}) {
-    const d = options.dampingFactor ?? 0.85;
-    const maxIterations = options.maxIterations ?? 40;
-    const epsilon = options.convergenceThreshold ?? 1e-5;
-    const soft404Threshold = options.soft404WeightThreshold ?? 0.8;
-
-    const allNodes = graph.getNodes();
-    const allEdges = graph.getEdges();
-
-    // 1. Filter Eligible Nodes
-    const eligibleNodes = allNodes.filter(node => {
-        if (node.noindex) return false;
-        if (node.isCollapsed) return false;
-        if (node.soft404Score && node.soft404Score > soft404Threshold) return false;
-        if (node.canonical && node.canonical !== node.url) return false;
-        if (node.status >= 400) return false; // Don't pass rank to broken pages
-        return true;
-    });
-
-    const nodeCount = eligibleNodes.length;
-    if (nodeCount === 0) return;
-
-    const nodeUrls = eligibleNodes.map(n => n.url);
-    const nodeMap = new Map<string, GraphNode>();
-    eligibleNodes.forEach(n => nodeMap.set(n.url, n));
-
-    // Initialize PageRank
-    let pr = new Map<string, number>();
-    nodeUrls.forEach(url => pr.set(url, 1 / nodeCount));
-
-    // Pre-calculate weighted outbound sums and inverted adjacency
-    const outWeights = new Map<string, number>();
-    const incoming = new Map<string, { source: string; weight: number }[]>();
-    const sinks: string[] = [];
-
-    // Initialize outWeights for all eligible nodes
-    nodeUrls.forEach(url => outWeights.set(url, 0));
-
-    for (const edge of allEdges) {
-        if (nodeMap.has(edge.source) && nodeMap.has(edge.target)) {
-            const weight = edge.weight || 1.0;
-
-            const sources = incoming.get(edge.target) ?? [];
-            sources.push({ source: edge.source, weight });
-            incoming.set(edge.target, sources);
-
-            outWeights.set(edge.source, (outWeights.get(edge.source) || 0) + weight);
-        }
-    }
-
-    // Identify sinks
-    nodeUrls.forEach(url => {
-        if ((outWeights.get(url) || 0) === 0) {
-            sinks.push(url);
-        }
-    });
-
-    // Iterative Calculation
-    for (let i = 0; i < maxIterations; i++) {
-        const nextPr = new Map<string, number>();
-
-        // Calculate total rank from sinks to redistribute
-        let sinkRankTotal = 0;
-        for (const url of sinks) {
-            sinkRankTotal += pr.get(url) || 0;
-        }
-
-        const baseRank = (1 - d) / nodeCount + (d * sinkRankTotal / nodeCount);
-
-        for (const url of nodeUrls) {
-            let rankFromLinks = 0;
-            const sources = incoming.get(url) || [];
-
-            for (const edge of sources) {
-                const sourceRank = pr.get(edge.source) || 0;
-                const sourceOutWeight = outWeights.get(edge.source) || 1.0;
-                rankFromLinks += sourceRank * (edge.weight / sourceOutWeight);
-            }
-
-            const newRank = baseRank + d * rankFromLinks;
-            nextPr.set(url, newRank);
-        }
-
-        // Convergence check
-        let maxDelta = 0;
-        for (const url of nodeUrls) {
-            const delta = Math.abs(nextPr.get(url)! - pr.get(url)!);
-            if (delta > maxDelta) maxDelta = delta;
-        }
-
-        pr = nextPr;
-
-        if (maxDelta < epsilon) break;
-    }
-
-    // 2. Normalization (0-100)
-    const ranks = Array.from(pr.values());
-    const minPR = Math.min(...ranks);
-    const maxPR = Math.max(...ranks);
-    const range = maxPR - minPR;
-
-    for (const node of eligibleNodes) {
-        const rawRank = pr.get(node.url)!;
-        node.pageRank = rawRank;
-
-        if (range > 1e-12) {
-            node.pageRankScore = 100 * (rawRank - minPR) / range;
-        } else {
-            // If there's no range, all eligible pages are equally important.
-            node.pageRankScore = 100;
-        }
-    }
-}

package/src/graph/simhash.ts

@@ -1,61 +0,0 @@
-export class SimHash {
-    private static FNV_PRIME = 1099511628211n;
-    private static FNV_OFFSET_BASIS = 14695981039346656037n;
-    private static MAX_UINT64 = 0xffffffffffffffffn;
-
-    /**
-     * Generates a 64-bit FNV-1a hash for a given string token.
-     */
-    static fnv1a64(token: string): bigint {
-        let hash = this.FNV_OFFSET_BASIS;
-        const len = token.length;
-        for (let i = 0; i < len; i++) {
-            hash ^= BigInt(token.charCodeAt(i));
-            // BigInt safe multiplication modulo 2^64
-            hash = (hash * this.FNV_PRIME) & this.MAX_UINT64;
-        }
-        return hash;
-    }
-
-    /**
-     * Generates a 64-bit SimHash from an array of tokens.
-     */
-    static generate(tokens: string[]): bigint {
-        const v = new Int32Array(64);
-
-        for (const token of tokens) {
-            const hash = this.fnv1a64(token);
-            for (let i = 0n; i < 64n; i++) {
-                const bit = (hash >> i) & 1n;
-                if (bit === 1n) {
-                    v[Number(i)]++;
-                } else {
-                    v[Number(i)]--;
-                }
-            }
-        }
-
-        let simhash = 0n;
-        for (let i = 0n; i < 64n; i++) {
-            if (v[Number(i)] > 0) {
-                simhash |= (1n << i);
-            }
-        }
-
-        return simhash;
-    }
-
-    /**
-     * Computes the Hamming distance between two 64-bit hashes.
-     */
-    static hammingDistance(a: bigint, b: bigint): number {
-        let xor = a ^ b;
-        let distance = 0;
-        while (xor > 0n) {
-            // Kernighan's bit counting
-            xor &= xor - 1n;
-            distance++;
-        }
-        return distance;
-    }
-}

package/src/index.ts

@@ -1,30 +0,0 @@
-export * from './crawler/crawl.js';
-export * from './crawler/metricsRunner.js';
-export * from './graph/metrics.js';
-export * from './report/html.js';
-export * from './report/sitegraph_template.js';
-export * from './report/sitegraphExport.js';
-export * from './graph/graph.js';
-export * from './diff/compare.js';
-export * from './scoring/orphanSeverity.js';
-export * from './graph/pagerank.js';
-export * from './graph/duplicate.js';
-export * from './graph/cluster.js';
-export * from './scoring/hits.js';
-export * from './analysis/analyze.js';
-export * from './analysis/content.js';
-export * from './analysis/seo.js';
-export * from './analysis/images.js';
-export * from './analysis/links.js';
-export * from './audit/index.js';
-export * from './audit/types.js';
-export * from './db/index.js';
-export * from './db/graphLoader.js';
-export * from './db/repositories/SiteRepository.js';
-export * from './db/repositories/SnapshotRepository.js';
-export * from './db/repositories/PageRepository.js';
-export * from './db/repositories/EdgeRepository.js';
-export * from './db/repositories/MetricsRepository.js';
-export * from './lock/lockManager.js';
-export * from './lock/hashKey.js';
-export * from './utils/version.js';

package/src/lock/hashKey.ts

@@ -1,51 +0,0 @@
-import crypto from 'node:crypto';
-import { normalizeUrl } from '../crawler/normalize.js';
-
-// Flags that change the nature of the crawl and should be part of the lock key
-const RELEVANT_FLAGS = [
-  'limit',
-  'depth',
-  'output',
-  'sitemap',
-  'incremental',
-  'detectSoft404',
-  'detectTraps',
-  'includeSubdomains',
-  'allow',
-  'deny',
-  'proxy',
-  'ua',
-  'maxRedirects',
-  'rate',
-  'maxBytes',
-  'concurrency'
-];
-
-export function generateLockKey(commandName: string, targetUrl: string, options: any): string {
-  // Respect the query stripping option consistent with sitegraph logic
-  const stripQuery = !options.query;
-
-  const normalizedTarget = normalizeUrl(targetUrl, '', { stripQuery }) || targetUrl;
-
-  // Extract relevant options in a deterministic order
-  const lockOptions: Record<string, any> = {};
-  for (const key of RELEVANT_FLAGS) {
-    if (options[key] !== undefined) {
-      lockOptions[key] = options[key];
-    }
-  }
-
-  // Create composite key object
-  const compositeKey = {
-    command: commandName,
-    target: normalizedTarget,
-    options: lockOptions
-  };
-
-  // Stringify and hash
-  // Since we inserted keys in a deterministic order (RELEVANT_FLAGS order),
-  // JSON.stringify will produce a stable string in V8/Node.js.
-  const stableString = JSON.stringify(compositeKey);
-
-  return crypto.createHash('sha256').update(stableString).digest('hex');
-}

package/src/lock/lockManager.ts

@@ -1,124 +0,0 @@
-import fs from 'node:fs/promises';
-import { existsSync, unlinkSync, readFileSync } from 'node:fs';
-import path from 'node:path';
-import os from 'node:os';
-import chalk from 'chalk';
-import { generateLockKey } from './hashKey.js';
-import { isPidAlive } from './pidCheck.js';
-
-interface LockData {
-  pid: number;
-  startedAt: number;
-  command: string;
-  target: string;
-  args: any;
-}
-
-export class LockManager {
-  private static lockFilePath: string | null = null;
-
-  private static get lockDir(): string {
-    return path.join(os.homedir(), '.crawlith', 'locks');
-  }
-
-  static async acquireLock(commandName: string, targetUrl: string, options: any, force: boolean = false): Promise<void> {
-    const lockHash = generateLockKey(commandName, targetUrl, options);
-
-    // Ensure lock directory exists
-    // We can use sync or async here. Since this is one-time setup, async is fine.
-    await fs.mkdir(this.lockDir, { recursive: true });
-
-    const lockPath = path.join(this.lockDir, `${lockHash}.lock`);
-
-    // Check existing lock
-    if (existsSync(lockPath)) {
-      let isStale: boolean;
-      let pid: number;
-
-      try {
-        const lockContent = readFileSync(lockPath, 'utf-8');
-        const lockData = JSON.parse(lockContent);
-        pid = lockData.pid;
-        isStale = !isPidAlive(pid);
-      } catch (_e) {
-        // Corrupted -> Treat as stale
-        isStale = true;
-        pid = 0; // Fallback, though unused if isStale is true
-      }
-
-      if (force) {
-        console.warn(chalk.yellow('Force mode enabled. Overriding existing lock.'));
-        try { unlinkSync(lockPath); } catch { /* ignore */ }
-      } else {
-        if (!isStale) {
-          console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (PID ${pid})`));
-          process.exit(1);
-        } else {
-          console.log(chalk.gray('Detected stale lock. Continuing execution.'));
-          try { unlinkSync(lockPath); } catch { /* ignore */ }
-        }
-      }
-    }
-
-    // Create new lock
-    try {
-      const data: LockData = {
-        pid: process.pid,
-        startedAt: Date.now(),
-        command: commandName,
-        target: targetUrl,
-        args: options
-      };
-
-      // 'wx' flag ensures atomic creation, failing if file exists
-      await fs.writeFile(lockPath, JSON.stringify(data, null, 2), { flag: 'wx', encoding: 'utf-8' });
-
-      this.lockFilePath = lockPath;
-      this.registerHandlers();
-    } catch (error: any) {
-      if (error.code === 'EEXIST') {
-        // Race condition: another process created lock between our check and open
-        console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (Race condition)`));
-        process.exit(1);
-      }
-      throw error;
-    }
-  }
-
-  static releaseLock(): void {
-    if (this.lockFilePath && existsSync(this.lockFilePath)) {
-      try {
-        unlinkSync(this.lockFilePath);
-        this.lockFilePath = null;
-      } catch (_error) {
-        // Ignore errors during cleanup
-      }
-    }
-  }
-
-  private static registerHandlers() {
-    // Ensure cleanup only happens once
-    const cleanup = () => {
-      this.releaseLock();
-    };
-
-    // process.on('exit') is only called when process.exit() is called or event loop empties.
-    // It requires synchronous cleanup.
-    process.on('exit', cleanup);
-
-    // Signals
-    process.on('SIGINT', () => {
-      cleanup();
-      process.exit(130);
-    });
-    process.on('SIGTERM', () => {
-      cleanup();
-      process.exit(143);
-    });
-    process.on('uncaughtException', (err) => {
-      console.error(chalk.red('Uncaught Exception:'), err);
-      cleanup();
-      process.exit(1);
-    });
-  }
-}

package/src/lock/pidCheck.ts

@@ -1,13 +0,0 @@
-export function isPidAlive(pid: number): boolean {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch (error: any) {
-    if (error.code === 'EPERM') {
-      // Process exists but no permission to signal -> Alive
-      return true;
-    }
-    // Process does not exist (ESRCH) or other error
-    return false;
-  }
-}