@crawlith/core 0.1.0

package/dist/graph/metrics.js

@@ -0,0 +1,74 @@
+export function calculateMetrics(graph, _maxDepth) {
+    const nodes = graph.getNodes();
+    const edges = graph.getEdges();
+    const totalPages = nodes.length;
+    const totalEdges = edges.length;
+    // Authority Score (per node)
+    const maxInLinks = nodes.reduce((max, n) => Math.max(max, n.inLinks), 0);
+    const getAuthority = (node) => {
+        if (maxInLinks === 0)
+            return 0;
+        return Math.log(1 + node.inLinks) / Math.log(1 + maxInLinks);
+    };
+    // orphanPages: inLinks === 0 && depth > 0
+    const orphanPages = nodes
+        .filter(n => n.inLinks === 0 && n.depth > 0)
+        .map(n => n.url);
+    // nearOrphans: inLinks === 1 && depth >= 3
+    const nearOrphans = nodes
+        .filter(n => n.inLinks === 1 && n.depth >= 3)
+        .map(n => n.url);
+    // deepPages: depth >= 4
+    const deepPages = nodes
+        .filter(n => n.depth >= 4) // Per requirement
+        .map(n => n.url);
+    // crawlEfficiencyScore: 1 - (deepPagesCount / totalPages)
+    const deepPagesCount = deepPages.length;
+    const crawlEfficiencyScore = totalPages > 0 ? 1 - (deepPagesCount / totalPages) : 1;
+    // averageDepth: sum(depth) / totalPages
+    const sumDepth = nodes.reduce((acc, n) => acc + n.depth, 0);
+    const averageDepth = totalPages > 0 ? sumDepth / totalPages : 0;
+    // structuralEntropy: Shannon entropy over outDegree distribution
+    const outDegreeCounts = new Map();
+    nodes.forEach(n => {
+        outDegreeCounts.set(n.outLinks, (outDegreeCounts.get(n.outLinks) || 0) + 1);
+    });
+    let structuralEntropy = 0;
+    if (totalPages > 0) {
+        for (const count of outDegreeCounts.values()) {
+            const p = count / totalPages;
+            if (p > 0) {
+                structuralEntropy -= p * Math.log2(p);
+            }
+        }
+    }
+    // topAuthorityPages: Top 10 by authority
+    const topAuthorityPages = [...nodes]
+        .map(n => ({ url: n.url, authority: n.authorityScore ?? getAuthority(n) }))
+        .sort((a, b) => b.authority - a.authority)
+        .slice(0, 10);
+    // topPageRankPages: Top 10 by raw PageRank
+    const topPageRankPages = [...nodes]
+        .filter(n => n.pageRank !== undefined)
+        .map(n => ({ url: n.url, score: n.pageRank }))
+        .sort((a, b) => b.score - a.score)
+        .slice(0, 10);
+    const averageOutDegree = totalPages > 0 ? totalEdges / totalPages : 0;
+    const maxDepthFound = nodes.reduce((max, n) => Math.max(max, n.depth), 0);
+    return {
+        totalPages,
+        totalEdges,
+        orphanPages,
+        nearOrphans,
+        deepPages,
+        topAuthorityPages,
+        averageOutDegree,
+        maxDepthFound,
+        crawlEfficiencyScore,
+        averageDepth,
+        structuralEntropy,
+        topPageRankPages,
+        limitReached: graph.limitReached,
+        sessionStats: graph.sessionStats
+    };
+}

package/dist/graph/pagerank.d.ts

@@ -0,0 +1,12 @@
+import { Graph } from './graph.js';
+interface PageRankOptions {
+    dampingFactor?: number;
+    maxIterations?: number;
+    convergenceThreshold?: number;
+    soft404WeightThreshold?: number;
+}
+/**
+ * Production-Grade Weighted PageRank Engine
+ */
+export declare function computePageRank(graph: Graph, options?: PageRankOptions): void;
+export {};

package/dist/graph/pagerank.js

@@ -0,0 +1,102 @@
+/**
+ * Production-Grade Weighted PageRank Engine
+ */
+export function computePageRank(graph, options = {}) {
+    const d = options.dampingFactor ?? 0.85;
+    const maxIterations = options.maxIterations ?? 40;
+    const epsilon = options.convergenceThreshold ?? 1e-5;
+    const soft404Threshold = options.soft404WeightThreshold ?? 0.8;
+    const allNodes = graph.getNodes();
+    const allEdges = graph.getEdges();
+    // 1. Filter Eligible Nodes
+    const eligibleNodes = allNodes.filter(node => {
+        if (node.noindex)
+            return false;
+        if (node.isCollapsed)
+            return false;
+        if (node.soft404Score && node.soft404Score > soft404Threshold)
+            return false;
+        if (node.canonical && node.canonical !== node.url)
+            return false;
+        if (node.status >= 400)
+            return false; // Don't pass rank to broken pages
+        return true;
+    });
+    const nodeCount = eligibleNodes.length;
+    if (nodeCount === 0)
+        return;
+    const nodeUrls = eligibleNodes.map(n => n.url);
+    const nodeMap = new Map();
+    eligibleNodes.forEach(n => nodeMap.set(n.url, n));
+    // Initialize PageRank
+    let pr = new Map();
+    nodeUrls.forEach(url => pr.set(url, 1 / nodeCount));
+    // Pre-calculate weighted outbound sums and inverted adjacency
+    const outWeights = new Map();
+    const incoming = new Map();
+    const sinks = [];
+    // Initialize outWeights for all eligible nodes
+    nodeUrls.forEach(url => outWeights.set(url, 0));
+    for (const edge of allEdges) {
+        if (nodeMap.has(edge.source) && nodeMap.has(edge.target)) {
+            const weight = edge.weight || 1.0;
+            const sources = incoming.get(edge.target) ?? [];
+            sources.push({ source: edge.source, weight });
+            incoming.set(edge.target, sources);
+            outWeights.set(edge.source, (outWeights.get(edge.source) || 0) + weight);
+        }
+    }
+    // Identify sinks
+    nodeUrls.forEach(url => {
+        if ((outWeights.get(url) || 0) === 0) {
+            sinks.push(url);
+        }
+    });
+    // Iterative Calculation
+    for (let i = 0; i < maxIterations; i++) {
+        const nextPr = new Map();
+        // Calculate total rank from sinks to redistribute
+        let sinkRankTotal = 0;
+        for (const url of sinks) {
+            sinkRankTotal += pr.get(url) || 0;
+        }
+        const baseRank = (1 - d) / nodeCount + (d * sinkRankTotal / nodeCount);
+        for (const url of nodeUrls) {
+            let rankFromLinks = 0;
+            const sources = incoming.get(url) || [];
+            for (const edge of sources) {
+                const sourceRank = pr.get(edge.source) || 0;
+                const sourceOutWeight = outWeights.get(edge.source) || 1.0;
+                rankFromLinks += sourceRank * (edge.weight / sourceOutWeight);
+            }
+            const newRank = baseRank + d * rankFromLinks;
+            nextPr.set(url, newRank);
+        }
+        // Convergence check
+        let maxDelta = 0;
+        for (const url of nodeUrls) {
+            const delta = Math.abs(nextPr.get(url) - pr.get(url));
+            if (delta > maxDelta)
+                maxDelta = delta;
+        }
+        pr = nextPr;
+        if (maxDelta < epsilon)
+            break;
+    }
+    // 2. Normalization (0-100)
+    const ranks = Array.from(pr.values());
+    const minPR = Math.min(...ranks);
+    const maxPR = Math.max(...ranks);
+    const range = maxPR - minPR;
+    for (const node of eligibleNodes) {
+        const rawRank = pr.get(node.url);
+        node.pageRank = rawRank;
+        if (range > 1e-12) {
+            node.pageRankScore = 100 * (rawRank - minPR) / range;
+        }
+        else {
+            // If there's no range, all eligible pages are equally important.
+            node.pageRankScore = 100;
+        }
+    }
+}

package/dist/graph/simhash.d.ts

@@ -0,0 +1,17 @@
+export declare class SimHash {
+    private static FNV_PRIME;
+    private static FNV_OFFSET_BASIS;
+    private static MAX_UINT64;
+    /**
+     * Generates a 64-bit FNV-1a hash for a given string token.
+     */
+    static fnv1a64(token: string): bigint;
+    /**
+     * Generates a 64-bit SimHash from an array of tokens.
+     */
+    static generate(tokens: string[]): bigint;
+    /**
+     * Computes the Hamming distance between two 64-bit hashes.
+     */
+    static hammingDistance(a: bigint, b: bigint): number;
+}

package/dist/graph/simhash.js

@@ -0,0 +1,56 @@
+export class SimHash {
+    static FNV_PRIME = 1099511628211n;
+    static FNV_OFFSET_BASIS = 14695981039346656037n;
+    static MAX_UINT64 = 0xffffffffffffffffn;
+    /**
+     * Generates a 64-bit FNV-1a hash for a given string token.
+     */
+    static fnv1a64(token) {
+        let hash = this.FNV_OFFSET_BASIS;
+        const len = token.length;
+        for (let i = 0; i < len; i++) {
+            hash ^= BigInt(token.charCodeAt(i));
+            // BigInt safe multiplication modulo 2^64
+            hash = (hash * this.FNV_PRIME) & this.MAX_UINT64;
+        }
+        return hash;
+    }
+    /**
+     * Generates a 64-bit SimHash from an array of tokens.
+     */
+    static generate(tokens) {
+        const v = new Int32Array(64);
+        for (const token of tokens) {
+            const hash = this.fnv1a64(token);
+            for (let i = 0n; i < 64n; i++) {
+                const bit = (hash >> i) & 1n;
+                if (bit === 1n) {
+                    v[Number(i)]++;
+                }
+                else {
+                    v[Number(i)]--;
+                }
+            }
+        }
+        let simhash = 0n;
+        for (let i = 0n; i < 64n; i++) {
+            if (v[Number(i)] > 0) {
+                simhash |= (1n << i);
+            }
+        }
+        return simhash;
+    }
+    /**
+     * Computes the Hamming distance between two 64-bit hashes.
+     */
+    static hammingDistance(a, b) {
+        let xor = a ^ b;
+        let distance = 0;
+        while (xor > 0n) {
+            // Kernighan's bit counting
+            xor &= xor - 1n;
+            distance++;
+        }
+        return distance;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,30 @@
+export * from './crawler/crawl.js';
+export * from './crawler/metricsRunner.js';
+export * from './graph/metrics.js';
+export * from './report/html.js';
+export * from './report/sitegraph_template.js';
+export * from './report/sitegraphExport.js';
+export * from './graph/graph.js';
+export * from './diff/compare.js';
+export * from './scoring/orphanSeverity.js';
+export * from './graph/pagerank.js';
+export * from './graph/duplicate.js';
+export * from './graph/cluster.js';
+export * from './scoring/hits.js';
+export * from './analysis/analyze.js';
+export * from './analysis/content.js';
+export * from './analysis/seo.js';
+export * from './analysis/images.js';
+export * from './analysis/links.js';
+export * from './audit/index.js';
+export * from './audit/types.js';
+export * from './db/index.js';
+export * from './db/graphLoader.js';
+export * from './db/repositories/SiteRepository.js';
+export * from './db/repositories/SnapshotRepository.js';
+export * from './db/repositories/PageRepository.js';
+export * from './db/repositories/EdgeRepository.js';
+export * from './db/repositories/MetricsRepository.js';
+export * from './lock/lockManager.js';
+export * from './lock/hashKey.js';
+export * from './utils/version.js';

package/dist/index.js

@@ -0,0 +1,30 @@
+export * from './crawler/crawl.js';
+export * from './crawler/metricsRunner.js';
+export * from './graph/metrics.js';
+export * from './report/html.js';
+export * from './report/sitegraph_template.js';
+export * from './report/sitegraphExport.js';
+export * from './graph/graph.js';
+export * from './diff/compare.js';
+export * from './scoring/orphanSeverity.js';
+export * from './graph/pagerank.js';
+export * from './graph/duplicate.js';
+export * from './graph/cluster.js';
+export * from './scoring/hits.js';
+export * from './analysis/analyze.js';
+export * from './analysis/content.js';
+export * from './analysis/seo.js';
+export * from './analysis/images.js';
+export * from './analysis/links.js';
+export * from './audit/index.js';
+export * from './audit/types.js';
+export * from './db/index.js';
+export * from './db/graphLoader.js';
+export * from './db/repositories/SiteRepository.js';
+export * from './db/repositories/SnapshotRepository.js';
+export * from './db/repositories/PageRepository.js';
+export * from './db/repositories/EdgeRepository.js';
+export * from './db/repositories/MetricsRepository.js';
+export * from './lock/lockManager.js';
+export * from './lock/hashKey.js';
+export * from './utils/version.js';

package/dist/lock/hashKey.d.ts

@@ -0,0 +1 @@
+export declare function generateLockKey(commandName: string, targetUrl: string, options: any): string;

package/dist/lock/hashKey.js

@@ -0,0 +1,44 @@
+import crypto from 'node:crypto';
+import { normalizeUrl } from '../crawler/normalize.js';
+// Flags that change the nature of the crawl and should be part of the lock key
+const RELEVANT_FLAGS = [
+    'limit',
+    'depth',
+    'output',
+    'sitemap',
+    'incremental',
+    'detectSoft404',
+    'detectTraps',
+    'includeSubdomains',
+    'allow',
+    'deny',
+    'proxy',
+    'ua',
+    'maxRedirects',
+    'rate',
+    'maxBytes',
+    'concurrency'
+];
+export function generateLockKey(commandName, targetUrl, options) {
+    // Respect the query stripping option consistent with sitegraph logic
+    const stripQuery = !options.query;
+    const normalizedTarget = normalizeUrl(targetUrl, '', { stripQuery }) || targetUrl;
+    // Extract relevant options in a deterministic order
+    const lockOptions = {};
+    for (const key of RELEVANT_FLAGS) {
+        if (options[key] !== undefined) {
+            lockOptions[key] = options[key];
+        }
+    }
+    // Create composite key object
+    const compositeKey = {
+        command: commandName,
+        target: normalizedTarget,
+        options: lockOptions
+    };
+    // Stringify and hash
+    // Since we inserted keys in a deterministic order (RELEVANT_FLAGS order),
+    // JSON.stringify will produce a stable string in V8/Node.js.
+    const stableString = JSON.stringify(compositeKey);
+    return crypto.createHash('sha256').update(stableString).digest('hex');
+}

package/dist/lock/lockManager.d.ts

@@ -0,0 +1,7 @@
+export declare class LockManager {
+    private static lockFilePath;
+    private static get lockDir();
+    static acquireLock(commandName: string, targetUrl: string, options: any, force?: boolean): Promise<void>;
+    static releaseLock(): void;
+    private static registerHandlers;
+}

package/dist/lock/lockManager.js

@@ -0,0 +1,112 @@
+import fs from 'node:fs/promises';
+import { existsSync, unlinkSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import chalk from 'chalk';
+import { generateLockKey } from './hashKey.js';
+import { isPidAlive } from './pidCheck.js';
+export class LockManager {
+    static lockFilePath = null;
+    static get lockDir() {
+        return path.join(os.homedir(), '.crawlith', 'locks');
+    }
+    static async acquireLock(commandName, targetUrl, options, force = false) {
+        const lockHash = generateLockKey(commandName, targetUrl, options);
+        // Ensure lock directory exists
+        // We can use sync or async here. Since this is one-time setup, async is fine.
+        await fs.mkdir(this.lockDir, { recursive: true });
+        const lockPath = path.join(this.lockDir, `${lockHash}.lock`);
+        // Check existing lock
+        if (existsSync(lockPath)) {
+            let isStale;
+            let pid;
+            try {
+                const lockContent = readFileSync(lockPath, 'utf-8');
+                const lockData = JSON.parse(lockContent);
+                pid = lockData.pid;
+                isStale = !isPidAlive(pid);
+            }
+            catch (_e) {
+                // Corrupted -> Treat as stale
+                isStale = true;
+                pid = 0; // Fallback, though unused if isStale is true
+            }
+            if (force) {
+                console.warn(chalk.yellow('Force mode enabled. Overriding existing lock.'));
+                try {
+                    unlinkSync(lockPath);
+                }
+                catch { /* ignore */ }
+            }
+            else {
+                if (!isStale) {
+                    console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (PID ${pid})`));
+                    process.exit(1);
+                }
+                else {
+                    console.log(chalk.gray('Detected stale lock. Continuing execution.'));
+                    try {
+                        unlinkSync(lockPath);
+                    }
+                    catch { /* ignore */ }
+                }
+            }
+        }
+        // Create new lock
+        try {
+            const data = {
+                pid: process.pid,
+                startedAt: Date.now(),
+                command: commandName,
+                target: targetUrl,
+                args: options
+            };
+            // 'wx' flag ensures atomic creation, failing if file exists
+            await fs.writeFile(lockPath, JSON.stringify(data, null, 2), { flag: 'wx', encoding: 'utf-8' });
+            this.lockFilePath = lockPath;
+            this.registerHandlers();
+        }
+        catch (error) {
+            if (error.code === 'EEXIST') {
+                // Race condition: another process created lock between our check and open
+                console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (Race condition)`));
+                process.exit(1);
+            }
+            throw error;
+        }
+    }
+    static releaseLock() {
+        if (this.lockFilePath && existsSync(this.lockFilePath)) {
+            try {
+                unlinkSync(this.lockFilePath);
+                this.lockFilePath = null;
+            }
+            catch (_error) {
+                // Ignore errors during cleanup
+            }
+        }
+    }
+    static registerHandlers() {
+        // Ensure cleanup only happens once
+        const cleanup = () => {
+            this.releaseLock();
+        };
+        // process.on('exit') is only called when process.exit() is called or event loop empties.
+        // It requires synchronous cleanup.
+        process.on('exit', cleanup);
+        // Signals
+        process.on('SIGINT', () => {
+            cleanup();
+            process.exit(130);
+        });
+        process.on('SIGTERM', () => {
+            cleanup();
+            process.exit(143);
+        });
+        process.on('uncaughtException', (err) => {
+            console.error(chalk.red('Uncaught Exception:'), err);
+            cleanup();
+            process.exit(1);
+        });
+    }
+}

package/dist/lock/pidCheck.d.ts

@@ -0,0 +1 @@
+export declare function isPidAlive(pid: number): boolean;

package/dist/lock/pidCheck.js

@@ -0,0 +1,14 @@
+export function isPidAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        if (error.code === 'EPERM') {
+            // Process exists but no permission to signal -> Alive
+            return true;
+        }
+        // Process does not exist (ESRCH) or other error
+        return false;
+    }
+}

package/dist/report/html.d.ts

@@ -0,0 +1,2 @@
+import { Metrics } from '../graph/metrics.js';
+export declare function generateHtml(graphData: any, metrics: Metrics): string;