@crawlith/core 0.1.0

package/src/graph/simhash.ts

@@ -0,0 +1,61 @@
+export class SimHash {
+    private static FNV_PRIME = 1099511628211n;
+    private static FNV_OFFSET_BASIS = 14695981039346656037n;
+    private static MAX_UINT64 = 0xffffffffffffffffn;
+
+    /**
+     * Generates a 64-bit FNV-1a hash for a given string token.
+     */
+    static fnv1a64(token: string): bigint {
+        let hash = this.FNV_OFFSET_BASIS;
+        const len = token.length;
+        for (let i = 0; i < len; i++) {
+            hash ^= BigInt(token.charCodeAt(i));
+            // BigInt safe multiplication modulo 2^64
+            hash = (hash * this.FNV_PRIME) & this.MAX_UINT64;
+        }
+        return hash;
+    }
+
+    /**
+     * Generates a 64-bit SimHash from an array of tokens.
+     */
+    static generate(tokens: string[]): bigint {
+        const v = new Int32Array(64);
+
+        for (const token of tokens) {
+            const hash = this.fnv1a64(token);
+            for (let i = 0n; i < 64n; i++) {
+                const bit = (hash >> i) & 1n;
+                if (bit === 1n) {
+                    v[Number(i)]++;
+                } else {
+                    v[Number(i)]--;
+                }
+            }
+        }
+
+        let simhash = 0n;
+        for (let i = 0n; i < 64n; i++) {
+            if (v[Number(i)] > 0) {
+                simhash |= (1n << i);
+            }
+        }
+
+        return simhash;
+    }
+
+    /**
+     * Computes the Hamming distance between two 64-bit hashes.
+     */
+    static hammingDistance(a: bigint, b: bigint): number {
+        let xor = a ^ b;
+        let distance = 0;
+        while (xor > 0n) {
+            // Kernighan's bit counting
+            xor &= xor - 1n;
+            distance++;
+        }
+        return distance;
+    }
+}

package/src/index.ts

@@ -0,0 +1,30 @@
+export * from './crawler/crawl.js';
+export * from './crawler/metricsRunner.js';
+export * from './graph/metrics.js';
+export * from './report/html.js';
+export * from './report/sitegraph_template.js';
+export * from './report/sitegraphExport.js';
+export * from './graph/graph.js';
+export * from './diff/compare.js';
+export * from './scoring/orphanSeverity.js';
+export * from './graph/pagerank.js';
+export * from './graph/duplicate.js';
+export * from './graph/cluster.js';
+export * from './scoring/hits.js';
+export * from './analysis/analyze.js';
+export * from './analysis/content.js';
+export * from './analysis/seo.js';
+export * from './analysis/images.js';
+export * from './analysis/links.js';
+export * from './audit/index.js';
+export * from './audit/types.js';
+export * from './db/index.js';
+export * from './db/graphLoader.js';
+export * from './db/repositories/SiteRepository.js';
+export * from './db/repositories/SnapshotRepository.js';
+export * from './db/repositories/PageRepository.js';
+export * from './db/repositories/EdgeRepository.js';
+export * from './db/repositories/MetricsRepository.js';
+export * from './lock/lockManager.js';
+export * from './lock/hashKey.js';
+export * from './utils/version.js';

package/src/lock/hashKey.ts

@@ -0,0 +1,51 @@
+import crypto from 'node:crypto';
+import { normalizeUrl } from '../crawler/normalize.js';
+
+// Flags that change the nature of the crawl and should be part of the lock key
+const RELEVANT_FLAGS = [
+  'limit',
+  'depth',
+  'output',
+  'sitemap',
+  'incremental',
+  'detectSoft404',
+  'detectTraps',
+  'includeSubdomains',
+  'allow',
+  'deny',
+  'proxy',
+  'ua',
+  'maxRedirects',
+  'rate',
+  'maxBytes',
+  'concurrency'
+];
+
+export function generateLockKey(commandName: string, targetUrl: string, options: any): string {
+  // Respect the query stripping option consistent with sitegraph logic
+  const stripQuery = !options.query;
+
+  const normalizedTarget = normalizeUrl(targetUrl, '', { stripQuery }) || targetUrl;
+
+  // Extract relevant options in a deterministic order
+  const lockOptions: Record<string, any> = {};
+  for (const key of RELEVANT_FLAGS) {
+    if (options[key] !== undefined) {
+      lockOptions[key] = options[key];
+    }
+  }
+
+  // Create composite key object
+  const compositeKey = {
+    command: commandName,
+    target: normalizedTarget,
+    options: lockOptions
+  };
+
+  // Stringify and hash
+  // Since we inserted keys in a deterministic order (RELEVANT_FLAGS order),
+  // JSON.stringify will produce a stable string in V8/Node.js.
+  const stableString = JSON.stringify(compositeKey);
+
+  return crypto.createHash('sha256').update(stableString).digest('hex');
+}

package/src/lock/lockManager.ts

@@ -0,0 +1,124 @@
+import fs from 'node:fs/promises';
+import { existsSync, unlinkSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import chalk from 'chalk';
+import { generateLockKey } from './hashKey.js';
+import { isPidAlive } from './pidCheck.js';
+
+interface LockData {
+  pid: number;
+  startedAt: number;
+  command: string;
+  target: string;
+  args: any;
+}
+
+export class LockManager {
+  private static lockFilePath: string | null = null;
+
+  private static get lockDir(): string {
+    return path.join(os.homedir(), '.crawlith', 'locks');
+  }
+
+  static async acquireLock(commandName: string, targetUrl: string, options: any, force: boolean = false): Promise<void> {
+    const lockHash = generateLockKey(commandName, targetUrl, options);
+
+    // Ensure lock directory exists
+    // We can use sync or async here. Since this is one-time setup, async is fine.
+    await fs.mkdir(this.lockDir, { recursive: true });
+
+    const lockPath = path.join(this.lockDir, `${lockHash}.lock`);
+
+    // Check existing lock
+    if (existsSync(lockPath)) {
+      let isStale: boolean;
+      let pid: number;
+
+      try {
+        const lockContent = readFileSync(lockPath, 'utf-8');
+        const lockData = JSON.parse(lockContent);
+        pid = lockData.pid;
+        isStale = !isPidAlive(pid);
+      } catch (_e) {
+        // Corrupted -> Treat as stale
+        isStale = true;
+        pid = 0; // Fallback, though unused if isStale is true
+      }
+
+      if (force) {
+        console.warn(chalk.yellow('Force mode enabled. Overriding existing lock.'));
+        try { unlinkSync(lockPath); } catch { /* ignore */ }
+      } else {
+        if (!isStale) {
+          console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (PID ${pid})`));
+          process.exit(1);
+        } else {
+          console.log(chalk.gray('Detected stale lock. Continuing execution.'));
+          try { unlinkSync(lockPath); } catch { /* ignore */ }
+        }
+      }
+    }
+
+    // Create new lock
+    try {
+      const data: LockData = {
+        pid: process.pid,
+        startedAt: Date.now(),
+        command: commandName,
+        target: targetUrl,
+        args: options
+      };
+
+      // 'wx' flag ensures atomic creation, failing if file exists
+      await fs.writeFile(lockPath, JSON.stringify(data, null, 2), { flag: 'wx', encoding: 'utf-8' });
+
+      this.lockFilePath = lockPath;
+      this.registerHandlers();
+    } catch (error: any) {
+      if (error.code === 'EEXIST') {
+        // Race condition: another process created lock between our check and open
+        console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (Race condition)`));
+        process.exit(1);
+      }
+      throw error;
+    }
+  }
+
+  static releaseLock(): void {
+    if (this.lockFilePath && existsSync(this.lockFilePath)) {
+      try {
+        unlinkSync(this.lockFilePath);
+        this.lockFilePath = null;
+      } catch (_error) {
+        // Ignore errors during cleanup
+      }
+    }
+  }
+
+  private static registerHandlers() {
+    // Ensure cleanup only happens once
+    const cleanup = () => {
+      this.releaseLock();
+    };
+
+    // process.on('exit') is only called when process.exit() is called or event loop empties.
+    // It requires synchronous cleanup.
+    process.on('exit', cleanup);
+
+    // Signals
+    process.on('SIGINT', () => {
+      cleanup();
+      process.exit(130);
+    });
+    process.on('SIGTERM', () => {
+      cleanup();
+      process.exit(143);
+    });
+    process.on('uncaughtException', (err) => {
+      console.error(chalk.red('Uncaught Exception:'), err);
+      cleanup();
+      process.exit(1);
+    });
+  }
+}

package/src/lock/pidCheck.ts

@@ -0,0 +1,13 @@
+export function isPidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error: any) {
+    if (error.code === 'EPERM') {
+      // Process exists but no permission to signal -> Alive
+      return true;
+    }
+    // Process does not exist (ESRCH) or other error
+    return false;
+  }
+}

package/src/report/html.ts

@@ -0,0 +1,227 @@
+import { Metrics } from '../graph/metrics.js';
+
+function safeJson(data: any): string {
+    return JSON.stringify(data).replace(/</g, '\\u003c');
+}
+
+export function generateHtml(graphData: any, metrics: Metrics): string {
+    const graphJson = safeJson(graphData);
+
+    return `<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Crawlith Site Graph</title>
+    <style>
+        body { margin: 0; overflow: hidden; font-family: sans-serif; }
+        #graph { width: 100vw; height: 100vh; background: #f0f0f0; }
+        .tooltip {
+            position: absolute;
+            background: white;
+            border: 1px solid #ccc;
+            padding: 10px;
+            pointer-events: none;
+            font-size: 12px;
+            box-shadow: 2px 2px 5px rgba(0,0,0,0.1);
+            display: none;
+        }
+        #metrics {
+            position: absolute;
+            top: 10px;
+            left: 10px;
+            background: rgba(255, 255, 255, 0.9);
+            padding: 15px;
+            border-radius: 5px;
+            box-shadow: 0 0 10px rgba(0,0,0,0.1);
+            max-width: 320px;
+            max-height: 90vh;
+            overflow-y: auto;
+            z-index: 100;
+        }
+        h1 { font-size: 18px; margin-top: 0; }
+        h2 { font-size: 14px; margin: 15px 0 5px; border-bottom: 1px solid #ddd; }
+        ul { padding-left: 20px; margin: 5px 0; }
+        .legend { margin-top: 10px; font-size: 11px; }
+        .legend-item { display: flex; align-items: center; margin-bottom: 3px; }
+        .dot { width: 8px; height: 8px; border-radius: 50%; margin-right: 5px; }
+        .stat-row { display: flex; justify-content: space-between; font-size: 13px; margin-bottom: 3px; }
+        .stat-label { color: #666; }
+        .stat-value { font-weight: bold; }
+    </style>
+</head>
+<body>
+    <div id="metrics">
+        <h1>Crawlith Site Graph</h1>
+        
+        <div class="stat-row">
+            <span class="stat-label">Discovered Pages:</span>
+            <span class="stat-value">${metrics.totalPages}</span>
+        </div>
+        ${metrics.sessionStats ? `
+        <div class="stat-row">
+            <span class="stat-label">Session Crawl:</span>
+            <span class="stat-value">${metrics.sessionStats.pagesFetched} pages</span>
+        </div>
+        ${metrics.sessionStats.pagesCached > 0 ? `
+        <div class="stat-row" style="font-size: 11px; margin-top: -3px;">
+            <span class="stat-label" style="padding-left: 10px;">- Reuse Cached:</span>
+            <span class="stat-value">${metrics.sessionStats.pagesCached}</span>
+        </div>` : ''}
+        ` : ''}
+        <div class="stat-row">
+            <span class="stat-label">Total Edges:</span>
+            <span class="stat-value">${metrics.totalEdges}</span>
+        </div>
+        <div class="stat-row">
+            <span class="stat-label">Max Depth:</span>
+            <span class="stat-value">${metrics.maxDepthFound}</span>
+        </div>
+        <div class="stat-row">
+            <span class="stat-label">Avg Out-Degree:</span>
+            <span class="stat-value">${metrics.averageOutDegree.toFixed(2)}</span>
+        </div>
+
+        <div class="legend">
+            <div class="legend-item"><div class="dot" style="background: red;"></div>Orphan (In-Links: 0)</div>
+            <div class="legend-item"><div class="dot" style="background: orange;"></div>Deep (Depth >= 4)</div>
+            <div class="legend-item"><div class="dot" style="background: blue;"></div>Normal</div>
+        </div>
+
+        ${metrics.topAuthorityPages.length > 0 ? `
+        <h3>Top Authority</h3>
+        <ul>
+            ${metrics.topAuthorityPages.map(p => `<li><a href="${p.url}" target="_blank">${new URL(p.url).pathname}</a> (${p.authority.toFixed(2)})</li>`).join('')}
+        </ul>
+        ` : ''}
+
+        ${metrics.orphanPages.length > 0 ? `
+        <h3>Orphan Pages (${metrics.orphanPages.length})</h3>
+        <details>
+            <summary>Show list</summary>
+            <ul>
+                ${metrics.orphanPages.slice(0, 20).map(url => `<li><a href="${url}" target="_blank">${url}</a></li>`).join('')}
+                ${metrics.orphanPages.length > 20 ? `<li>... and ${metrics.orphanPages.length - 20} more</li>` : ''}
+            </ul>
+        </details>
+        ` : ''}
+    </div>
+    <div id="graph"></div>
+    <div class="tooltip" id="tooltip"></div>
+
+    <script src="https://d3js.org/d3.v7.min.js"></script>
+    <script>
+        // Make data available globally
+        window.GRAPH_DATA = ${graphJson};
+        
+        const data = window.GRAPH_DATA;
+        const width = window.innerWidth;
+        const height = window.innerHeight;
+
+        const svg = d3.select("#graph").append("svg")
+            .attr("width", width)
+            .attr("height", height)
+            .call(d3.zoom().on("zoom", (event) => {
+                g.attr("transform", event.transform);
+            }));
+
+        const g = svg.append("g");
+        
+        // Define arrow marker
+        svg.append("defs").selectAll("marker")
+            .data(["arrow"])
+            .enter().append("marker")
+            .attr("id", d => d)
+            .attr("viewBox", "0 -5 10 10")
+            .attr("refX", 15)
+            .attr("refY", 0)
+            .attr("markerWidth", 6)
+            .attr("markerHeight", 6)
+            .attr("orient", "auto")
+            .append("path")
+            .attr("d", "M0,-5L10,0L0,5")
+            .attr("fill", "#999");
+
+        const simulation = d3.forceSimulation(data.nodes)
+            .force("link", d3.forceLink(data.edges).id(d => d.url).distance(100))
+            .force("charge", d3.forceManyBody().strength(-300))
+            .force("center", d3.forceCenter(width / 2, height / 2))
+            .force("collide", d3.forceCollide().radius(d => Math.sqrt((d.inLinks || 0) + 1) * 5 + 2));
+
+        const link = g.append("g")
+            .attr("stroke", "#999")
+            .attr("stroke-opacity", 0.6)
+            .selectAll("line")
+            .data(data.edges)
+            .join("line")
+            .attr("stroke-width", 1)
+            .attr("marker-end", "url(#arrow)");
+
+
+        const node = g.append("g")
+            .attr("stroke", "#fff")
+            .attr("stroke-width", 1.5)
+            .selectAll("circle")
+            .data(data.nodes)
+            .join("circle")
+            .attr("r", d => Math.sqrt((d.inLinks || 0) + 1) * 3 + 2)
+            .attr("fill", d => {
+                if (d.inLinks === 0 && d.depth > 0) return "red";
+                if (d.depth >= 4) return "orange";
+                return "blue";
+            })
+            .call(d3.drag()
+                .on("start", dragstarted)
+                .on("drag", dragged)
+                .on("end", dragended));
+
+        const tooltip = d3.select("#tooltip");
+
+        node.on("mouseover", (event, d) => {
+            tooltip.style("display", "block")
+                .html(\`
+                    <strong>URL:</strong> \${d.url}<br>
+                    <strong>Depth:</strong> \${d.depth}<br>
+                    <strong>In-Links:</strong> \${d.inLinks}<br>
+                    <strong>Out-Links:</strong> \${d.outLinks}<br>
+                    <strong>Status:</strong> \${d.status}
+                \`)
+                .style("left", (event.pageX + 10) + "px")
+                .style("top", (event.pageY - 10) + "px");
+        })
+        .on("mouseout", () => {
+            tooltip.style("display", "none");
+        });
+
+        simulation.on("tick", () => {
+            link
+                .attr("x1", d => d.source.x)
+                .attr("y1", d => d.source.y)
+                .attr("x2", d => d.target.x)
+                .attr("y2", d => d.target.y);
+
+            node
+                .attr("cx", d => d.x)
+                .attr("cy", d => d.y);
+        });
+
+        function dragstarted(event, d) {
+            if (!event.active) simulation.alphaTarget(0.3).restart();
+            d.fx = d.x;
+            d.fy = d.y;
+        }
+
+        function dragged(event, d) {
+            d.fx = event.x;
+            d.fy = event.y;
+        }
+
+        function dragended(event, d) {
+            if (!event.active) simulation.alphaTarget(0);
+            d.fx = null;
+            d.fy = null;
+        }
+    </script>
+</body>
+</html>`;
+}

package/src/report/sitegraphExport.ts

@@ -0,0 +1,58 @@
+export function renderSitegraphCsvNodes(graphData: any): string {
+    const nodeHeaders = ['URL', 'Depth', 'Status', 'InboundLinks', 'OutboundLinks', 'PageRankScore'];
+    const nodeRows = graphData.nodes.map((n: any) => {
+        const outbound = graphData.edges.filter((e: any) => e.source === n.url).length;
+        const inbound = graphData.edges.filter((e: any) => e.target === n.url).length;
+        const statusStr = n.status === 0 ? 'Pending/Limit' : n.status;
+        return [n.url, n.depth, statusStr, inbound, outbound, (n.pageRankScore || 0).toFixed(3)].join(',');
+    });
+    return [nodeHeaders.join(','), ...nodeRows].join('\n');
+}
+
+export function renderSitegraphCsvEdges(graphData: any): string {
+    const edgeHeaders = ['Source', 'Target', 'Weight'];
+    const edgeRows = graphData.edges.map((e: any) => [e.source, e.target, e.weight].join(','));
+    return [edgeHeaders.join(','), ...edgeRows].join('\n');
+}
+
+export function renderSitegraphMarkdown(url: string, graphData: any, metrics: any, graph: any): string {
+    const md = [
+        `# Crawlith Crawl Summary - ${url}`,
+        '',
+        `## 📊 Metrics`,
+        `- Total Pages Discovered: ${metrics.totalPages}`,
+        `- Session Pages Crawled: ${graph.sessionStats?.pagesFetched ?? 0}`,
+        `- Total Edges: ${metrics.totalEdges}`,
+        `- Avg Depth: ${metrics.averageDepth.toFixed(2)}`,
+        `- Max Depth: ${metrics.maxDepthFound}`,
+        `- Crawl Efficiency: ${(metrics.crawlEfficiencyScore * 100).toFixed(1)}%`,
+        '',
+        `## 📄 Top Pages (by In-degree)`,
+    ];
+
+    const topPages = [...graphData.nodes]
+        .map((n: any) => ({ ...n, inLinks: graphData.edges.filter((e: any) => e.target === n.url).length }))
+        .sort((a, b) => b.inLinks - a.inLinks)
+        .slice(0, 10);
+
+    md.push('| URL | Inbound | Status |');
+    md.push('| :--- | :--- | :--- |');
+    topPages.forEach(p => {
+        const statusStr = p.status === 0 ? 'Pending/Limit' : p.status;
+        md.push(`| ${p.url} | ${p.inLinks} | ${statusStr} |`);
+    });
+
+    if (metrics.topPageRankPages?.length > 0) {
+        md.push('');
+        md.push('## 🏆 Top PageRank Pages');
+        md.push('| URL | Score |');
+        md.push('| :--- | :--- |');
+        metrics.topPageRankPages.slice(0, 10).forEach((p: any) => {
+            const node = graph.nodes?.get ? graph.nodes.get(p.url) : graph.getNodes?.().find((x: any) => x.url === p.url);
+            const score = node?.pageRankScore ?? 0;
+            md.push(`| ${p.url} | ${score.toFixed(3)}/100 |`);
+        });
+    }
+
+    return md.join('\n');
+}