@crawlith/core 0.1.0 → 0.1.1

package/src/graph/graph.ts

@@ -35,6 +35,11 @@ export interface GraphNode {
   clusterId?: number;
   bytesReceived?: number;
   linkRole?: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
+  crawlStatus?: string;
+  wordCount?: number;
+  thinContentScore?: number;
+  externalLinkRatio?: number;
+  orphanScore?: number;
 }
 
 export interface GraphEdge {
@@ -60,7 +65,7 @@ export interface CrawlStats {
 
 export class Graph {
   nodes: Map<string, GraphNode> = new Map();
-  // Using string "source|target" to ensure uniqueness efficiently. Mapping to weight.
+  // Using JSON string of [source, target] to ensure uniqueness. Mapping to weight.
   edges: Map<string, number> = new Map();
   limitReached: boolean = false;
   sessionStats: CrawlStats = {
@@ -73,6 +78,21 @@ export class Graph {
   duplicateClusters: { id: string; type: 'exact' | 'near' | 'template_heavy'; size: number; representative: string; severity: 'low' | 'medium' | 'high' }[] = [];
   contentClusters: ClusterInfo[] = [];
 
+  /**
+   * Generates a unique key for an edge.
+   */
+  static getEdgeKey(source: string, target: string): string {
+    return JSON.stringify([source, target]);
+  }
+
+  /**
+   * Parses an edge key back into source and target.
+   */
+  static parseEdgeKey(key: string): { source: string; target: string } {
+    const [source, target] = JSON.parse(key);
+    return { source, target };
+  }
+
   /**
    * Adds a node to the graph if it doesn't exist.
    * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
@@ -113,7 +133,7 @@ export class Graph {
     const targetNode = this.nodes.get(target);
 
     if (sourceNode && targetNode) {
-      const edgeKey = `${source}|${target}`;
+      const edgeKey = Graph.getEdgeKey(source, target);
       if (!this.edges.has(edgeKey)) {
         this.edges.set(edgeKey, weight);
         sourceNode.outLinks++;
@@ -134,7 +154,7 @@ export class Graph {
 
   getEdges(): GraphEdge[] {
     return Array.from(this.edges.entries()).map(([edge, weight]) => {
-      const [source, target] = edge.split('|');
+      const { source, target } = Graph.parseEdgeKey(edge);
       return { source, target, weight };
     });
   }
@@ -157,7 +177,7 @@ export class Graph {
     }
     if (json.edges) {
       for (const edge of json.edges) {
-        const key = `${edge.source}|${edge.target}`;
+        const key = Graph.getEdgeKey(edge.source, edge.target);
         graph.edges.set(key, edge.weight || 1.0);
       }
     }

package/src/graph/metrics.ts

@@ -29,6 +29,21 @@ export function calculateMetrics(graph: Graph, _maxDepth: number): Metrics {
   const totalPages = nodes.length;
   const totalEdges = edges.length;
 
+  // Identify broken nodes
+  const brokenNodes = new Set(nodes.filter(n => n.status >= 400 || n.status === 0).map(n => n.url));
+
+  // Populate brokenLinks per node
+  for (const node of nodes) {
+    const nodeEdges = edges.filter(e => e.source === node.url);
+    const broken = nodeEdges
+      .map(e => e.target)
+      .filter(targetUrl => brokenNodes.has(targetUrl));
+
+    if (broken.length > 0) {
+      node.brokenLinks = broken;
+    }
+  }
+
   // Authority Score (per node)
   const maxInLinks = nodes.reduce((max, n) => Math.max(max, n.inLinks), 0);
   const getAuthority = (node: GraphNode) => {

package/src/graph/pagerank.ts

@@ -26,6 +26,7 @@ export function computePageRank(graph: Graph, options: PageRankOptions = {}) {
         if (node.soft404Score && node.soft404Score > soft404Threshold) return false;
         if (node.canonical && node.canonical !== node.url) return false;
         if (node.status >= 400) return false; // Don't pass rank to broken pages
+        if (node.status === 0) return false; // Don't pass rank to uncrawled/external pages
         return true;
     });
 

package/src/graph/simhash.ts

@@ -2,6 +2,8 @@ export class SimHash {
     private static FNV_PRIME = 1099511628211n;
     private static FNV_OFFSET_BASIS = 14695981039346656037n;
     private static MAX_UINT64 = 0xffffffffffffffffn;
+    public static readonly BANDS = 4;
+    public static readonly BAND_WIDTH = 16;
 
     /**
      * Generates a 64-bit FNV-1a hash for a given string token.
@@ -45,6 +47,19 @@ export class SimHash {
         return simhash;
     }
 
+    /**
+     * Splits a 64-bit SimHash into 4 bands of 16 bits.
+     */
+    static getBands(simhash: bigint): number[] {
+        const bands: number[] = [];
+        for (let i = 0; i < SimHash.BANDS; i++) {
+            // Extract 16-bit chunks
+            const chunk = Number((simhash >> BigInt(i * SimHash.BAND_WIDTH)) & 0xFFFFn);
+            bands.push(chunk);
+        }
+        return bands;
+    }
+
     /**
      * Computes the Hamming distance between two 64-bit hashes.
      */

package/src/index.ts

@@ -1,15 +1,17 @@
 export * from './crawler/crawl.js';
+export * from './crawler/normalize.js';
 export * from './crawler/metricsRunner.js';
 export * from './graph/metrics.js';
 export * from './report/html.js';
-export * from './report/sitegraph_template.js';
-export * from './report/sitegraphExport.js';
+export * from './report/crawl_template.js';
+export * from './report/crawlExport.js';
 export * from './graph/graph.js';
 export * from './diff/compare.js';
 export * from './scoring/orphanSeverity.js';
 export * from './graph/pagerank.js';
 export * from './graph/duplicate.js';
 export * from './graph/cluster.js';
+export * from './scoring/health.js';
 export * from './scoring/hits.js';
 export * from './analysis/analyze.js';
 export * from './analysis/content.js';
@@ -28,3 +30,4 @@ export * from './db/repositories/MetricsRepository.js';
 export * from './lock/lockManager.js';
 export * from './lock/hashKey.js';
 export * from './utils/version.js';
+export * from './events.js';

package/src/lock/hashKey.ts

@@ -22,7 +22,7 @@ const RELEVANT_FLAGS = [
 ];
 
 export function generateLockKey(commandName: string, targetUrl: string, options: any): string {
-  // Respect the query stripping option consistent with sitegraph logic
+  // Respect the query stripping option consistent with crawl logic
   const stripQuery = !options.query;
 
   const normalizedTarget = normalizeUrl(targetUrl, '', { stripQuery }) || targetUrl;

package/src/lock/lockManager.ts

@@ -2,9 +2,9 @@ import fs from 'node:fs/promises';
 import { existsSync, unlinkSync, readFileSync } from 'node:fs';
 import path from 'node:path';
 import os from 'node:os';
-import chalk from 'chalk';
 import { generateLockKey } from './hashKey.js';
 import { isPidAlive } from './pidCheck.js';
+import { EngineContext } from '../events.js';
 
 interface LockData {
   pid: number;
@@ -16,16 +16,17 @@ interface LockData {
 
 export class LockManager {
   private static lockFilePath: string | null = null;
+  private static context: EngineContext | null = null;
 
   private static get lockDir(): string {
     return path.join(os.homedir(), '.crawlith', 'locks');
   }
 
-  static async acquireLock(commandName: string, targetUrl: string, options: any, force: boolean = false): Promise<void> {
+  static async acquireLock(commandName: string, targetUrl: string, options: any, context?: EngineContext, force: boolean = false): Promise<void> {
+    this.context = context || null;
     const lockHash = generateLockKey(commandName, targetUrl, options);
 
     // Ensure lock directory exists
-    // We can use sync or async here. Since this is one-time setup, async is fine.
     await fs.mkdir(this.lockDir, { recursive: true });
 
     const lockPath = path.join(this.lockDir, `${lockHash}.lock`);
@@ -43,18 +44,18 @@ export class LockManager {
       } catch (_e) {
         // Corrupted -> Treat as stale
         isStale = true;
-        pid = 0; // Fallback, though unused if isStale is true
+        pid = 0;
       }
 
       if (force) {
-        console.warn(chalk.yellow('Force mode enabled. Overriding existing lock.'));
+        this.log('warn', 'Force mode enabled. Overriding existing lock.');
         try { unlinkSync(lockPath); } catch { /* ignore */ }
       } else {
         if (!isStale) {
-          console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (PID ${pid})`));
+          this.log('error', `Crawlith: command already running for ${targetUrl} (PID ${pid})`);
           process.exit(1);
         } else {
-          console.log(chalk.gray('Detected stale lock. Continuing execution.'));
+          this.log('info', 'Detected stale lock. Continuing execution.');
           try { unlinkSync(lockPath); } catch { /* ignore */ }
         }
       }
@@ -77,8 +78,7 @@ export class LockManager {
       this.registerHandlers();
     } catch (error: any) {
       if (error.code === 'EEXIST') {
-        // Race condition: another process created lock between our check and open
-        console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (Race condition)`));
+        this.log('error', `Crawlith: command already running for ${targetUrl} (Race condition)`);
         process.exit(1);
       }
       throw error;
@@ -96,17 +96,25 @@ export class LockManager {
     }
   }
 
+  private static log(type: 'info' | 'warn' | 'error', message: string, error?: unknown) {
+    if (this.context) {
+      this.context.emit({ type, message, error });
+    } else {
+      // Fallback for legacy usage or when no context provided
+      if (type === 'error') console.error(message, error || '');
+      else if (type === 'warn') console.warn(message);
+      else console.log(message);
+    }
+  }
+
   private static registerHandlers() {
     // Ensure cleanup only happens once
     const cleanup = () => {
       this.releaseLock();
     };
 
-    // process.on('exit') is only called when process.exit() is called or event loop empties.
-    // It requires synchronous cleanup.
     process.on('exit', cleanup);
 
-    // Signals
     process.on('SIGINT', () => {
       cleanup();
       process.exit(130);
@@ -116,7 +124,7 @@ export class LockManager {
       process.exit(143);
     });
     process.on('uncaughtException', (err) => {
-      console.error(chalk.red('Uncaught Exception:'), err);
+      this.log('error', 'Uncaught Exception', err);
       cleanup();
       process.exit(1);
     });