@crawlith/core 0.1.0 → 0.1.2

package/dist/events.d.ts

@@ -0,0 +1,56 @@
+export type CrawlEvent = {
+    type: 'crawl:start';
+    url: string;
+} | {
+    type: 'crawl:success';
+    url: string;
+    status: number;
+    durationMs: number;
+    depth?: number;
+} | {
+    type: 'crawl:error';
+    url: string;
+    error: string;
+    depth?: number;
+} | {
+    type: 'crawl:limit-reached';
+    limit: number;
+} | {
+    type: 'crawl:progress';
+    pagesCrawled: number;
+    queued: number;
+    active: number;
+    nodesFound: number;
+    edgesFound: number;
+    phase?: string;
+} | {
+    type: 'queue:enqueue';
+    url: string;
+    depth: number;
+} | {
+    type: 'metrics:start';
+    phase: string;
+} | {
+    type: 'metrics:complete';
+    durationMs: number;
+} | {
+    type: 'debug';
+    message: string;
+    context?: unknown;
+} | {
+    type: 'info';
+    message: string;
+    context?: unknown;
+} | {
+    type: 'warn';
+    message: string;
+    context?: unknown;
+} | {
+    type: 'error';
+    message: string;
+    error?: unknown;
+    context?: unknown;
+};
+export interface EngineContext {
+    emit: (event: CrawlEvent) => void;
+}

package/dist/events.js

@@ -0,0 +1 @@
+export {};

package/dist/graph/graph.d.ts

@@ -1,5 +1,6 @@
 export interface GraphNode {
     url: string;
+    isInternal?: boolean;
     depth: number;
     inLinks: number;
     outLinks: number;
@@ -9,45 +10,48 @@ export interface GraphNode {
     nofollow?: boolean;
     brokenLinks?: string[];
     redirectChain?: string[];
+    discoveredViaSitemap?: boolean;
     incrementalStatus?: 'new' | 'changed' | 'unchanged' | 'deleted';
     etag?: string;
     lastModified?: string;
     contentHash?: string;
     html?: string;
-    pageRank?: number;
-    pageRankScore?: number;
-    authorityScore?: number;
-    hubScore?: number;
-    duplicateClusterId?: string;
-    duplicateType?: 'exact' | 'near' | 'template_heavy' | 'none';
-    isClusterPrimary?: boolean;
-    isCollapsed?: boolean;
-    collapseInto?: string;
     simhash?: string;
     uniqueTokenRatio?: number;
-    soft404Score?: number;
-    soft404Signals?: string[];
     crawlTrapFlag?: boolean;
     crawlTrapRisk?: number;
     trapType?: string;
     securityError?: string;
     retries?: number;
-    clusterId?: number;
     bytesReceived?: number;
-    linkRole?: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
+    crawlStatus?: string;
+    wordCount?: number;
+    thinContentScore?: number;
+    externalLinkRatio?: number;
+    h1Count?: number;
+    h2Count?: number;
+    title?: string;
+    clusterId?: number;
+    duplicateClusterId?: string;
+    duplicateType?: 'exact' | 'near' | 'template_heavy';
+    pagerankScore?: number;
+    hubScore?: number;
+    authScore?: number;
+    linkRole?: string;
+    soft404Score?: number;
+    headingScore?: number;
+    orphanScore?: number;
+    orphanType?: string;
+    impactLevel?: string;
+    headingData?: string;
+    isClusterPrimary?: boolean;
+    isCollapsed?: boolean;
 }
 export interface GraphEdge {
     source: string;
     target: string;
     weight: number;
 }
-export interface ClusterInfo {
-    id: number;
-    count: number;
-    primaryUrl: string;
-    risk: 'low' | 'medium' | 'high';
-    sharedPathPrefix?: string;
-}
 export interface CrawlStats {
     pagesFetched: number;
     pagesCached: number;
@@ -59,25 +63,23 @@ export declare class Graph {
     edges: Map<string, number>;
     limitReached: boolean;
     sessionStats: CrawlStats;
-    trapClusters: {
-        pattern: string;
-        type: string;
-        count: number;
-    }[];
-    duplicateClusters: {
-        id: string;
-        type: 'exact' | 'near' | 'template_heavy';
-        size: number;
-        representative: string;
-        severity: 'low' | 'medium' | 'high';
-    }[];
-    contentClusters: ClusterInfo[];
+    /**
+     * Generates a unique key for an edge.
+     */
+    static getEdgeKey(source: string, target: string): string;
+    /**
+     * Parses an edge key back into source and target.
+     */
+    static parseEdgeKey(key: string): {
+        source: string;
+        target: string;
+    };
     /**
      * Adds a node to the graph if it doesn't exist.
      * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
      * Depth is only set on creation (BFS guarantees shortest path first).
      */
-    addNode(url: string, depth: number, status?: number): void;
+    addNode(url: string, depth: number, status?: number, isInternal?: boolean): void;
     updateNodeData(url: string, data: Partial<GraphNode>): void;
     /**
      * Adds a directed edge between two nodes.
@@ -90,14 +92,6 @@ export declare class Graph {
     toJSON(): {
         nodes: GraphNode[];
         edges: GraphEdge[];
-        duplicateClusters: {
-            id: string;
-            type: "exact" | "near" | "template_heavy";
-            size: number;
-            representative: string;
-            severity: "low" | "medium" | "high";
-        }[];
-        contentClusters: ClusterInfo[];
     };
     static fromJSON(json: any): Graph;
 }

package/dist/graph/graph.js

@@ -1,6 +1,6 @@
 export class Graph {
     nodes = new Map();
-    // Using string "source|target" to ensure uniqueness efficiently. Mapping to weight.
+    // Using JSON string of [source, target] to ensure uniqueness. Mapping to weight.
     edges = new Map();
     limitReached = false;
     sessionStats = {
@@ -9,19 +9,33 @@ export class Graph {
         pagesSkipped: 0,
         totalFound: 0
     };
-    trapClusters = [];
-    duplicateClusters = [];
-    contentClusters = [];
+    /**
+     * Generates a unique key for an edge.
+     */
+    static getEdgeKey(source, target) {
+        return source + '\x00' + target;
+    }
+    /**
+     * Parses an edge key back into source and target.
+     */
+    static parseEdgeKey(key) {
+        const splitIndex = key.indexOf('\x00');
+        return {
+            source: key.slice(0, splitIndex),
+            target: key.slice(splitIndex + 1)
+        };
+    }
     /**
      * Adds a node to the graph if it doesn't exist.
      * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
      * Depth is only set on creation (BFS guarantees shortest path first).
      */
-    addNode(url, depth, status = 0) {
+    addNode(url, depth, status = 0, isInternal = true) {
         const existing = this.nodes.get(url);
         if (!existing) {
             this.nodes.set(url, {
                 url,
+                isInternal,
                 depth,
                 status,
                 inLinks: 0,
@@ -33,6 +47,9 @@ export class Graph {
             if (status !== 0) {
                 existing.status = status;
             }
+            if (isInternal !== undefined) {
+                existing.isInternal = isInternal;
+            }
         }
     }
     updateNodeData(url, data) {
@@ -50,7 +67,7 @@ export class Graph {
         const sourceNode = this.nodes.get(source);
         const targetNode = this.nodes.get(target);
         if (sourceNode && targetNode) {
-            const edgeKey = `${source}|${target}`;
+            const edgeKey = Graph.getEdgeKey(source, target);
             if (!this.edges.has(edgeKey)) {
                 this.edges.set(edgeKey, weight);
                 sourceNode.outLinks++;
@@ -70,16 +87,14 @@ export class Graph {
     }
     getEdges() {
         return Array.from(this.edges.entries()).map(([edge, weight]) => {
-            const [source, target] = edge.split('|');
+            const { source, target } = Graph.parseEdgeKey(edge);
             return { source, target, weight };
         });
     }
     toJSON() {
         return {
             nodes: this.getNodes(),
-            edges: this.getEdges(),
-            duplicateClusters: this.duplicateClusters,
-            contentClusters: this.contentClusters
+            edges: this.getEdges()
         };
     }
     static fromJSON(json) {
@@ -91,16 +106,10 @@ export class Graph {
         }
         if (json.edges) {
             for (const edge of json.edges) {
-                const key = `${edge.source}|${edge.target}`;
+                const key = Graph.getEdgeKey(edge.source, edge.target);
                 graph.edges.set(key, edge.weight || 1.0);
             }
         }
-        if (json.duplicateClusters) {
-            graph.duplicateClusters = json.duplicateClusters;
-        }
-        if (json.contentClusters) {
-            graph.contentClusters = json.contentClusters;
-        }
         return graph;
     }
 }

package/dist/graph/hits.d.ts

@@ -0,0 +1,23 @@
+import { Graph } from './graph.js';
+export type LinkRole = 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
+export interface HITSRow {
+    authority_score: number;
+    hub_score: number;
+    link_role: LinkRole;
+}
+export interface HITSOptions {
+    iterations?: number;
+}
+/**
+ * Service to compute Hub and Authority scores using the HITS algorithm.
+ * Operates purely on the internal link graph.
+ */
+export declare class HITSService {
+    /**
+     * Computes Hub and Authority scores using the HITS algorithm.
+     * @param {Graph} graph - The link graph to analyze.
+     * @param {HITSOptions} options - Algorithm options (e.g. number of iterations).
+     * @returns {Map<string, HITSRow>} A map of page URLs to their HITS results.
+     */
+    evaluate(graph: Graph, options?: HITSOptions): Map<string, HITSRow>;
+}

package/dist/graph/hits.js

@@ -0,0 +1,111 @@
+/**
+ * Service to compute Hub and Authority scores using the HITS algorithm.
+ * Operates purely on the internal link graph.
+ */
+export class HITSService {
+    /**
+     * Computes Hub and Authority scores using the HITS algorithm.
+     * @param {Graph} graph - The link graph to analyze.
+     * @param {HITSOptions} options - Algorithm options (e.g. number of iterations).
+     * @returns {Map<string, HITSRow>} A map of page URLs to their HITS results.
+     */
+    evaluate(graph, options = {}) {
+        const iterations = options.iterations || 20;
+        const nodes = graph.getNodes();
+        // 1. Filter eligible nodes
+        const eligibleNodes = nodes.filter(n => (n.status === 200 || n.status === 0) &&
+            (!n.redirectChain || n.redirectChain.length === 0) &&
+            !n.noindex);
+        const N = eligibleNodes.length;
+        const results = new Map();
+        if (N === 0)
+            return results;
+        // Map URL to Index for O(1) access
+        const urlToIndex = new Map();
+        for (let i = 0; i < N; i++) {
+            urlToIndex.set(eligibleNodes[i].url, i);
+        }
+        // Build Adjacency Lists
+        const incoming = new Array(N).fill(null).map(() => []);
+        const outgoing = new Array(N).fill(null).map(() => []);
+        const allEdges = graph.getEdges();
+        for (const edge of allEdges) {
+            if (edge.source === edge.target)
+                continue;
+            const sourceIndex = urlToIndex.get(edge.source);
+            const targetIndex = urlToIndex.get(edge.target);
+            if (sourceIndex !== undefined && targetIndex !== undefined) {
+                const weight = edge.weight || 1.0;
+                incoming[targetIndex].push({ sourceIndex, weight });
+                outgoing[sourceIndex].push({ targetIndex, weight });
+            }
+        }
+        // Initialize Scores
+        const authScores = new Float64Array(N).fill(1.0);
+        const hubScores = new Float64Array(N).fill(1.0);
+        // 2. Iteration
+        for (let iter = 0; iter < iterations; iter++) {
+            let normAuth = 0;
+            for (let i = 0; i < N; i++) {
+                const inLinks = incoming[i];
+                let newAuth = 0;
+                for (let j = 0; j < inLinks.length; j++) {
+                    const link = inLinks[j];
+                    newAuth += hubScores[link.sourceIndex] * link.weight;
+                }
+                authScores[i] = newAuth;
+                normAuth += newAuth * newAuth;
+            }
+            normAuth = Math.sqrt(normAuth);
+            if (normAuth > 0) {
+                for (let i = 0; i < N; i++)
+                    authScores[i] /= normAuth;
+            }
+            let normHub = 0;
+            for (let i = 0; i < N; i++) {
+                const outLinks = outgoing[i];
+                let newHub = 0;
+                for (let j = 0; j < outLinks.length; j++) {
+                    const link = outLinks[j];
+                    newHub += authScores[link.targetIndex] * link.weight;
+                }
+                hubScores[i] = newHub;
+                normHub += newHub * newHub;
+            }
+            normHub = Math.sqrt(normHub);
+            if (normHub > 0) {
+                for (let i = 0; i < N; i++)
+                    hubScores[i] /= normHub;
+            }
+        }
+        // 3. Classification and Result Mapping
+        const sortedAuth = [...authScores].sort((a, b) => a - b);
+        const sortedHub = [...hubScores].sort((a, b) => a - b);
+        const medianAuth = sortedAuth[Math.floor(sortedAuth.length / 2)];
+        const medianHub = sortedHub[Math.floor(sortedHub.length / 2)];
+        const maxAuth = sortedAuth[sortedAuth.length - 1];
+        const maxHub = sortedHub[sortedHub.length - 1];
+        for (let i = 0; i < N; i++) {
+            const auth = authScores[i];
+            const hub = hubScores[i];
+            const url = eligibleNodes[i].url;
+            const isHighAuth = (auth > medianAuth || (auth === maxAuth && auth > 0)) && auth > 0.00001;
+            const isHighHub = (hub > medianHub || (hub === maxHub && hub > 0)) && hub > 0.00001;
+            let link_role = 'peripheral';
+            if (isHighAuth && isHighHub)
+                link_role = 'power';
+            else if (isHighAuth)
+                link_role = 'authority';
+            else if (isHighHub)
+                link_role = 'hub';
+            else if (auth > 0.00001 && hub > 0.00001)
+                link_role = 'balanced';
+            results.set(url, {
+                authority_score: auth,
+                hub_score: hub,
+                link_role
+            });
+        }
+        return results;
+    }
+}

package/dist/graph/metrics.d.ts

@@ -14,10 +14,6 @@ export interface Metrics {
     crawlEfficiencyScore: number;
     averageDepth: number;
     structuralEntropy: number;
-    topPageRankPages: {
-        url: string;
-        score: number;
-    }[];
     limitReached: boolean;
     sessionStats?: {
         pagesFetched: number;

package/dist/graph/metrics.js

@@ -3,6 +3,28 @@ export function calculateMetrics(graph, _maxDepth) {
     const edges = graph.getEdges();
     const totalPages = nodes.length;
     const totalEdges = edges.length;
+    // Identify broken nodes
+    const brokenNodes = new Set(nodes.filter(n => n.status >= 400 || n.status === 0).map(n => n.url));
+    // Pre-compute outgoing edges per node for faster lookup
+    const outgoingEdges = new Map();
+    for (const edge of edges) {
+        let targets = outgoingEdges.get(edge.source);
+        if (!targets) {
+            targets = [];
+            outgoingEdges.set(edge.source, targets);
+        }
+        targets.push(edge.target);
+    }
+    // Populate brokenLinks per node
+    for (const node of nodes) {
+        const targets = outgoingEdges.get(node.url);
+        if (targets) {
+            const broken = targets.filter(targetUrl => brokenNodes.has(targetUrl));
+            if (broken.length > 0) {
+                node.brokenLinks = broken;
+            }
+        }
+    }
     // Authority Score (per node)
     const maxInLinks = nodes.reduce((max, n) => Math.max(max, n.inLinks), 0);
     const getAuthority = (node) => {
@@ -43,16 +65,11 @@ export function calculateMetrics(graph, _maxDepth) {
         }
     }
     // topAuthorityPages: Top 10 by authority
-    const topAuthorityPages = [...nodes]
-        .map(n => ({ url: n.url, authority: n.authorityScore ?? getAuthority(n) }))
+    const topAuthorityPages = nodes
+        .filter(n => n.isInternal !== false && n.status > 0)
+        .map(n => ({ url: n.url, authority: getAuthority(n) }))
         .sort((a, b) => b.authority - a.authority)
         .slice(0, 10);
-    // topPageRankPages: Top 10 by raw PageRank
-    const topPageRankPages = [...nodes]
-        .filter(n => n.pageRank !== undefined)
-        .map(n => ({ url: n.url, score: n.pageRank }))
-        .sort((a, b) => b.score - a.score)
-        .slice(0, 10);
     const averageOutDegree = totalPages > 0 ? totalEdges / totalPages : 0;
     const maxDepthFound = nodes.reduce((max, n) => Math.max(max, n.depth), 0);
     return {
@@ -67,7 +84,6 @@ export function calculateMetrics(graph, _maxDepth) {
         crawlEfficiencyScore,
         averageDepth,
         structuralEntropy,
-        topPageRankPages,
         limitReached: graph.limitReached,
         sessionStats: graph.sessionStats
     };

package/dist/graph/pagerank.d.ts

@@ -1,12 +1,25 @@
 import { Graph } from './graph.js';
-interface PageRankOptions {
+export interface PageRankRow {
+    raw_rank: number;
+    score: number;
+}
+export interface PageRankOptions {
     dampingFactor?: number;
     maxIterations?: number;
     convergenceThreshold?: number;
     soft404WeightThreshold?: number;
+    neutralScoreWhenFlat?: number;
 }
 /**
- * Production-Grade Weighted PageRank Engine
+ * Service to analyze a site's link graph and compute PageRank metrics.
+ * Runs only on the full crawl graph.
  */
-export declare function computePageRank(graph: Graph, options?: PageRankOptions): void;
-export {};
+export declare class PageRankService {
+    /**
+     * Computes a Production-Grade Weighted PageRank over the given graph.
+     * @param {Graph} graph - The full site graph structure.
+     * @param {PageRankOptions} options - Configuration overrides for damping factor, limits, etc.
+     * @returns {Map<string, PageRankRow>} The individual metrics keyed by exact normalized url.
+     */
+    evaluate(graph: Graph, options?: PageRankOptions): Map<string, PageRankRow>;
+}