@crawlith/core 0.1.0 → 0.1.1

package/dist/graph/graph.d.ts

@@ -35,6 +35,11 @@ export interface GraphNode {
     clusterId?: number;
     bytesReceived?: number;
     linkRole?: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
+    crawlStatus?: string;
+    wordCount?: number;
+    thinContentScore?: number;
+    externalLinkRatio?: number;
+    orphanScore?: number;
 }
 export interface GraphEdge {
     source: string;
@@ -72,6 +77,17 @@ export declare class Graph {
         severity: 'low' | 'medium' | 'high';
     }[];
     contentClusters: ClusterInfo[];
+    /**
+     * Generates a unique key for an edge.
+     */
+    static getEdgeKey(source: string, target: string): string;
+    /**
+     * Parses an edge key back into source and target.
+     */
+    static parseEdgeKey(key: string): {
+        source: string;
+        target: string;
+    };
     /**
      * Adds a node to the graph if it doesn't exist.
      * If it exists, updates the status if the new status is non-zero (meaning we crawled it).

package/dist/graph/graph.js

@@ -1,6 +1,6 @@
 export class Graph {
     nodes = new Map();
-    // Using string "source|target" to ensure uniqueness efficiently. Mapping to weight.
+    // Using JSON string of [source, target] to ensure uniqueness. Mapping to weight.
     edges = new Map();
     limitReached = false;
     sessionStats = {
@@ -12,6 +12,19 @@ export class Graph {
     trapClusters = [];
     duplicateClusters = [];
     contentClusters = [];
+    /**
+     * Generates a unique key for an edge.
+     */
+    static getEdgeKey(source, target) {
+        return JSON.stringify([source, target]);
+    }
+    /**
+     * Parses an edge key back into source and target.
+     */
+    static parseEdgeKey(key) {
+        const [source, target] = JSON.parse(key);
+        return { source, target };
+    }
     /**
      * Adds a node to the graph if it doesn't exist.
      * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
@@ -50,7 +63,7 @@ export class Graph {
         const sourceNode = this.nodes.get(source);
         const targetNode = this.nodes.get(target);
         if (sourceNode && targetNode) {
-            const edgeKey = `${source}|${target}`;
+            const edgeKey = Graph.getEdgeKey(source, target);
             if (!this.edges.has(edgeKey)) {
                 this.edges.set(edgeKey, weight);
                 sourceNode.outLinks++;
@@ -70,7 +83,7 @@ export class Graph {
     }
     getEdges() {
         return Array.from(this.edges.entries()).map(([edge, weight]) => {
-            const [source, target] = edge.split('|');
+            const { source, target } = Graph.parseEdgeKey(edge);
             return { source, target, weight };
         });
     }
@@ -91,7 +104,7 @@ export class Graph {
         }
         if (json.edges) {
             for (const edge of json.edges) {
-                const key = `${edge.source}|${edge.target}`;
+                const key = Graph.getEdgeKey(edge.source, edge.target);
                 graph.edges.set(key, edge.weight || 1.0);
             }
         }

package/dist/graph/metrics.js

@@ -3,6 +3,18 @@ export function calculateMetrics(graph, _maxDepth) {
     const edges = graph.getEdges();
     const totalPages = nodes.length;
     const totalEdges = edges.length;
+    // Identify broken nodes
+    const brokenNodes = new Set(nodes.filter(n => n.status >= 400 || n.status === 0).map(n => n.url));
+    // Populate brokenLinks per node
+    for (const node of nodes) {
+        const nodeEdges = edges.filter(e => e.source === node.url);
+        const broken = nodeEdges
+            .map(e => e.target)
+            .filter(targetUrl => brokenNodes.has(targetUrl));
+        if (broken.length > 0) {
+            node.brokenLinks = broken;
+        }
+    }
     // Authority Score (per node)
     const maxInLinks = nodes.reduce((max, n) => Math.max(max, n.inLinks), 0);
     const getAuthority = (node) => {

package/dist/graph/pagerank.js

@@ -20,6 +20,8 @@ export function computePageRank(graph, options = {}) {
             return false;
         if (node.status >= 400)
             return false; // Don't pass rank to broken pages
+        if (node.status === 0)
+            return false; // Don't pass rank to uncrawled/external pages
         return true;
     });
     const nodeCount = eligibleNodes.length;

package/dist/graph/simhash.d.ts

@@ -2,6 +2,8 @@ export declare class SimHash {
     private static FNV_PRIME;
     private static FNV_OFFSET_BASIS;
     private static MAX_UINT64;
+    static readonly BANDS = 4;
+    static readonly BAND_WIDTH = 16;
     /**
      * Generates a 64-bit FNV-1a hash for a given string token.
      */
@@ -10,6 +12,10 @@ export declare class SimHash {
      * Generates a 64-bit SimHash from an array of tokens.
      */
     static generate(tokens: string[]): bigint;
+    /**
+     * Splits a 64-bit SimHash into 4 bands of 16 bits.
+     */
+    static getBands(simhash: bigint): number[];
     /**
      * Computes the Hamming distance between two 64-bit hashes.
      */

package/dist/graph/simhash.js

@@ -2,6 +2,8 @@ export class SimHash {
     static FNV_PRIME = 1099511628211n;
     static FNV_OFFSET_BASIS = 14695981039346656037n;
     static MAX_UINT64 = 0xffffffffffffffffn;
+    static BANDS = 4;
+    static BAND_WIDTH = 16;
     /**
      * Generates a 64-bit FNV-1a hash for a given string token.
      */
@@ -40,6 +42,18 @@ export class SimHash {
         }
         return simhash;
     }
+    /**
+     * Splits a 64-bit SimHash into 4 bands of 16 bits.
+     */
+    static getBands(simhash) {
+        const bands = [];
+        for (let i = 0; i < SimHash.BANDS; i++) {
+            // Extract 16-bit chunks
+            const chunk = Number((simhash >> BigInt(i * SimHash.BAND_WIDTH)) & 0xffffn);
+            bands.push(chunk);
+        }
+        return bands;
+    }
     /**
      * Computes the Hamming distance between two 64-bit hashes.
      */

package/dist/index.d.ts

@@ -1,15 +1,17 @@
 export * from './crawler/crawl.js';
+export * from './crawler/normalize.js';
 export * from './crawler/metricsRunner.js';
 export * from './graph/metrics.js';
 export * from './report/html.js';
-export * from './report/sitegraph_template.js';
-export * from './report/sitegraphExport.js';
+export * from './report/crawl_template.js';
+export * from './report/crawlExport.js';
 export * from './graph/graph.js';
 export * from './diff/compare.js';
 export * from './scoring/orphanSeverity.js';
 export * from './graph/pagerank.js';
 export * from './graph/duplicate.js';
 export * from './graph/cluster.js';
+export * from './scoring/health.js';
 export * from './scoring/hits.js';
 export * from './analysis/analyze.js';
 export * from './analysis/content.js';
@@ -28,3 +30,4 @@ export * from './db/repositories/MetricsRepository.js';
 export * from './lock/lockManager.js';
 export * from './lock/hashKey.js';
 export * from './utils/version.js';
+export * from './events.js';

package/dist/index.js

@@ -1,15 +1,17 @@
 export * from './crawler/crawl.js';
+export * from './crawler/normalize.js';
 export * from './crawler/metricsRunner.js';
 export * from './graph/metrics.js';
 export * from './report/html.js';
-export * from './report/sitegraph_template.js';
-export * from './report/sitegraphExport.js';
+export * from './report/crawl_template.js';
+export * from './report/crawlExport.js';
 export * from './graph/graph.js';
 export * from './diff/compare.js';
 export * from './scoring/orphanSeverity.js';
 export * from './graph/pagerank.js';
 export * from './graph/duplicate.js';
 export * from './graph/cluster.js';
+export * from './scoring/health.js';
 export * from './scoring/hits.js';
 export * from './analysis/analyze.js';
 export * from './analysis/content.js';
@@ -28,3 +30,4 @@ export * from './db/repositories/MetricsRepository.js';
 export * from './lock/lockManager.js';
 export * from './lock/hashKey.js';
 export * from './utils/version.js';
+export * from './events.js';

package/dist/lock/hashKey.js

@@ -20,7 +20,7 @@ const RELEVANT_FLAGS = [
     'concurrency'
 ];
 export function generateLockKey(commandName, targetUrl, options) {
-    // Respect the query stripping option consistent with sitegraph logic
+    // Respect the query stripping option consistent with crawl logic
     const stripQuery = !options.query;
     const normalizedTarget = normalizeUrl(targetUrl, '', { stripQuery }) || targetUrl;
     // Extract relevant options in a deterministic order

package/dist/lock/lockManager.d.ts

@@ -1,7 +1,10 @@
+import { EngineContext } from '../events.js';
 export declare class LockManager {
     private static lockFilePath;
+    private static context;
     private static get lockDir();
-    static acquireLock(commandName: string, targetUrl: string, options: any, force?: boolean): Promise<void>;
+    static acquireLock(commandName: string, targetUrl: string, options: any, context?: EngineContext, force?: boolean): Promise<void>;
     static releaseLock(): void;
+    private static log;
     private static registerHandlers;
 }

package/dist/lock/lockManager.js

@@ -2,18 +2,18 @@ import fs from 'node:fs/promises';
 import { existsSync, unlinkSync, readFileSync } from 'node:fs';
 import path from 'node:path';
 import os from 'node:os';
-import chalk from 'chalk';
 import { generateLockKey } from './hashKey.js';
 import { isPidAlive } from './pidCheck.js';
 export class LockManager {
     static lockFilePath = null;
+    static context = null;
     static get lockDir() {
         return path.join(os.homedir(), '.crawlith', 'locks');
     }
-    static async acquireLock(commandName, targetUrl, options, force = false) {
+    static async acquireLock(commandName, targetUrl, options, context, force = false) {
+        this.context = context || null;
         const lockHash = generateLockKey(commandName, targetUrl, options);
         // Ensure lock directory exists
-        // We can use sync or async here. Since this is one-time setup, async is fine.
         await fs.mkdir(this.lockDir, { recursive: true });
         const lockPath = path.join(this.lockDir, `${lockHash}.lock`);
         // Check existing lock
@@ -29,10 +29,10 @@ export class LockManager {
             catch (_e) {
                 // Corrupted -> Treat as stale
                 isStale = true;
-                pid = 0; // Fallback, though unused if isStale is true
+                pid = 0;
             }
             if (force) {
-                console.warn(chalk.yellow('Force mode enabled. Overriding existing lock.'));
+                this.log('warn', 'Force mode enabled. Overriding existing lock.');
                 try {
                     unlinkSync(lockPath);
                 }
@@ -40,11 +40,11 @@ export class LockManager {
             }
             else {
                 if (!isStale) {
-                    console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (PID ${pid})`));
+                    this.log('error', `Crawlith: command already running for ${targetUrl} (PID ${pid})`);
                     process.exit(1);
                 }
                 else {
-                    console.log(chalk.gray('Detected stale lock. Continuing execution.'));
+                    this.log('info', 'Detected stale lock. Continuing execution.');
                     try {
                         unlinkSync(lockPath);
                     }
@@ -68,8 +68,7 @@ export class LockManager {
         }
         catch (error) {
             if (error.code === 'EEXIST') {
-                // Race condition: another process created lock between our check and open
-                console.error(chalk.red(`Crawlith: command already running for ${targetUrl} (Race condition)`));
+                this.log('error', `Crawlith: command already running for ${targetUrl} (Race condition)`);
                 process.exit(1);
             }
             throw error;
@@ -86,15 +85,26 @@ export class LockManager {
             }
         }
     }
+    static log(type, message, error) {
+        if (this.context) {
+            this.context.emit({ type, message, error });
+        }
+        else {
+            // Fallback for legacy usage or when no context provided
+            if (type === 'error')
+                console.error(message, error || '');
+            else if (type === 'warn')
+                console.warn(message);
+            else
+                console.log(message);
+        }
+    }
     static registerHandlers() {
         // Ensure cleanup only happens once
         const cleanup = () => {
             this.releaseLock();
         };
-        // process.on('exit') is only called when process.exit() is called or event loop empties.
-        // It requires synchronous cleanup.
         process.on('exit', cleanup);
-        // Signals
         process.on('SIGINT', () => {
             cleanup();
             process.exit(130);
@@ -104,7 +114,7 @@ export class LockManager {
             process.exit(143);
         });
         process.on('uncaughtException', (err) => {
-            console.error(chalk.red('Uncaught Exception:'), err);
+            this.log('error', 'Uncaught Exception', err);
             cleanup();
             process.exit(1);
         });