@crawlith/core 0.1.0

package/dist/scoring/hits.js

@@ -0,0 +1,111 @@
+/**
+ * Computes Hub and Authority scores using the HITS algorithm.
+ * Operates purely on the internal link graph.
+ */
+export function computeHITS(graph, options = {}) {
+    const iterations = options.iterations || 20;
+    const nodes = graph.getNodes();
+    // 1. Filter eligible nodes
+    // Eligibility: status 200, non-redirect (redirectChain empty), not noindex, non-external
+    const eligibleNodes = nodes.filter(n => n.status === 200 &&
+        (!n.redirectChain || n.redirectChain.length === 0) &&
+        !n.noindex);
+    if (eligibleNodes.length === 0)
+        return;
+    const urlToNode = new Map();
+    for (const node of eligibleNodes) {
+        urlToNode.set(node.url, node);
+        // 2. Initialization
+        node.authorityScore = 1.0;
+        node.hubScore = 1.0;
+    }
+    const allEdges = graph.getEdges();
+    // Filter edges: internal links only (both source and target must be in eligibleNodes), no self-links
+    const eligibleEdges = allEdges.filter(e => e.source !== e.target &&
+        urlToNode.has(e.source) &&
+        urlToNode.has(e.target));
+    // Group edges for efficient iteration
+    const incoming = new Map();
+    const outgoing = new Map();
+    for (const edge of eligibleEdges) {
+        if (!incoming.has(edge.target))
+            incoming.set(edge.target, []);
+        incoming.get(edge.target).push({ source: edge.source, weight: edge.weight });
+        if (!outgoing.has(edge.source))
+            outgoing.set(edge.source, []);
+        outgoing.get(edge.source).push({ target: edge.target, weight: edge.weight });
+    }
+    // 3. Iteration
+    for (let i = 0; i < iterations; i++) {
+        // Update Authorities
+        let normAuth = 0;
+        for (const node of eligibleNodes) {
+            const inLinks = incoming.get(node.url) || [];
+            let newAuth = 0;
+            for (const link of inLinks) {
+                const sourceNode = urlToNode.get(link.source);
+                newAuth += (sourceNode.hubScore || 0) * link.weight;
+            }
+            node.authorityScore = newAuth;
+            normAuth += newAuth * newAuth;
+        }
+        // Normalize Authorities (L2 norm)
+        normAuth = Math.sqrt(normAuth);
+        if (normAuth > 0) {
+            for (const node of eligibleNodes) {
+                node.authorityScore = (node.authorityScore || 0) / normAuth;
+            }
+        }
+        // Update Hubs
+        let normHub = 0;
+        for (const node of eligibleNodes) {
+            const outLinks = outgoing.get(node.url) || [];
+            let newHub = 0;
+            for (const link of outLinks) {
+                const targetNode = urlToNode.get(link.target);
+                newHub += (targetNode.authorityScore || 0) * link.weight;
+            }
+            node.hubScore = newHub;
+            normHub += newHub * newHub;
+        }
+        // Normalize Hubs (L2 norm)
+        normHub = Math.sqrt(normHub);
+        if (normHub > 0) {
+            for (const node of eligibleNodes) {
+                node.hubScore = (node.hubScore || 0) / normHub;
+            }
+        }
+    }
+    // 4. Classification Logic
+    classifyLinkRoles(eligibleNodes);
+}
+function classifyLinkRoles(nodes) {
+    if (nodes.length === 0)
+        return;
+    const authScores = nodes.map(n => n.authorityScore || 0).sort((a, b) => a - b);
+    const hubScores = nodes.map(n => n.hubScore || 0).sort((a, b) => a - b);
+    // Use 75th percentile as "high" threshold
+    const medianAuth = authScores[Math.floor(authScores.length / 2)];
+    const medianHub = hubScores[Math.floor(hubScores.length / 2)];
+    for (const node of nodes) {
+        const auth = node.authorityScore || 0;
+        const hub = node.hubScore || 0;
+        const isHighAuth = auth > medianAuth && auth > 0.0001;
+        const isHighHub = hub > medianHub && hub > 0.0001;
+        if (isHighAuth && isHighHub) {
+            node.linkRole = 'power';
+        }
+        else if (isHighAuth) {
+            node.linkRole = 'authority';
+        }
+        else if (isHighHub) {
+            node.linkRole = 'hub';
+        }
+        else if (auth > 0.0001 && hub > 0.0001) {
+            node.linkRole = 'balanced';
+        }
+        else {
+            node.linkRole = 'peripheral';
+        }
+    }
+}

package/dist/scoring/orphanSeverity.d.ts

@@ -0,0 +1,39 @@
+export type OrphanType = 'hard' | 'near' | 'soft' | 'crawl-only';
+export type ImpactLevel = 'low' | 'medium' | 'high' | 'critical';
+export interface SitegraphNode {
+    url: string;
+    depth: number;
+    inLinks: number;
+    outLinks: number;
+    status: number;
+    discoveredViaSitemap?: boolean;
+    robotsExcluded?: boolean;
+    canonicalUrl?: string;
+    isHomepage?: boolean;
+    wordCount?: number;
+    hasStructuredData?: boolean;
+    pageType?: string;
+    noindex?: boolean;
+    duplicateContent?: boolean;
+    isProductOrCommercial?: boolean;
+}
+export interface SitegraphEdge {
+    source: string;
+    target: string;
+}
+export interface OrphanScoringOptions {
+    enabled: boolean;
+    severityEnabled: boolean;
+    includeSoftOrphans: boolean;
+    minInbound: number;
+    rootUrl?: string;
+}
+export type AnnotatedNode = SitegraphNode & {
+    orphan: boolean;
+    orphanType?: OrphanType;
+    orphanSeverity?: number;
+    impactLevel?: ImpactLevel;
+};
+export declare function mapImpactLevel(score: number): ImpactLevel;
+export declare function calculateOrphanSeverity(orphanType: OrphanType, node: SitegraphNode): number;
+export declare function annotateOrphans(nodes: SitegraphNode[], edges: SitegraphEdge[], options: OrphanScoringOptions): AnnotatedNode[];

package/dist/scoring/orphanSeverity.js

@@ -0,0 +1,125 @@
+const LOW_VALUE_PATTERNS = [
+    /[?&](page|p)=\d+/i,
+    /\/(page|tag|tags|category|categories)\//i,
+    /[?&](q|query|search|filter|sort)=/i,
+    /\/search(\/|\?|$)/i
+];
+function isLowValuePage(node) {
+    const type = (node.pageType || '').toLowerCase();
+    if (['pagination', 'tag', 'category', 'filter', 'search', 'archive'].includes(type)) {
+        return true;
+    }
+    if (node.noindex) {
+        return true;
+    }
+    return LOW_VALUE_PATTERNS.some((pattern) => pattern.test(node.url));
+}
+function clampScore(score) {
+    return Math.max(0, Math.min(100, Math.round(score)));
+}
+export function mapImpactLevel(score) {
+    if (score <= 39)
+        return 'low';
+    if (score <= 69)
+        return 'medium';
+    if (score <= 89)
+        return 'high';
+    return 'critical';
+}
+export function calculateOrphanSeverity(orphanType, node) {
+    let score = 0;
+    switch (orphanType) {
+        case 'hard':
+            score = 90;
+            break;
+        case 'crawl-only':
+            score = 80;
+            break;
+        case 'near':
+            score = node.inLinks <= 1 ? 70 : 60;
+            break;
+        case 'soft':
+            score = 50;
+            break;
+    }
+    let positiveModifier = 0;
+    if ((node.wordCount || 0) > 800)
+        positiveModifier += 10;
+    if (node.hasStructuredData)
+        positiveModifier += 10;
+    if (node.depth <= 2)
+        positiveModifier += 10;
+    if (node.isProductOrCommercial)
+        positiveModifier += 10;
+    positiveModifier = Math.min(20, positiveModifier);
+    let negativeModifier = 0;
+    if ((node.wordCount || 0) > 0 && (node.wordCount || 0) < 300)
+        negativeModifier += 20;
+    if (node.noindex)
+        negativeModifier += 20;
+    if (node.duplicateContent)
+        negativeModifier += 20;
+    if ((node.pageType || '').toLowerCase() === 'archive' || (node.pageType || '').toLowerCase() === 'pagination')
+        negativeModifier += 20;
+    negativeModifier = Math.min(20, negativeModifier);
+    score += positiveModifier;
+    score -= negativeModifier;
+    return clampScore(score);
+}
+function consolidateInboundByCanonical(nodes) {
+    const canonicalInbound = new Map();
+    for (const node of nodes) {
+        const canonical = node.canonicalUrl || node.url;
+        canonicalInbound.set(canonical, (canonicalInbound.get(canonical) || 0) + node.inLinks);
+    }
+    return canonicalInbound;
+}
+export function annotateOrphans(nodes, edges, options) {
+    if (!options.enabled) {
+        return nodes.map((node) => ({ ...node, orphan: false }));
+    }
+    const canonicalInbound = consolidateInboundByCanonical(nodes);
+    const nodeByUrl = new Map(nodes.map((node) => [node.url, node]));
+    return nodes.map((node) => {
+        const isHomepage = node.isHomepage || (options.rootUrl ? node.url === options.rootUrl : node.depth === 0);
+        if (isHomepage || node.robotsExcluded) {
+            return { ...node, orphan: false };
+        }
+        const canonical = node.canonicalUrl || node.url;
+        const inbound = canonicalInbound.get(canonical) || 0;
+        let orphanType;
+        if (inbound === 0) {
+            orphanType = node.discoveredViaSitemap ? 'crawl-only' : 'hard';
+        }
+        else if (inbound <= options.minInbound) {
+            orphanType = 'near';
+        }
+        if (!orphanType && options.includeSoftOrphans && inbound > 0) {
+            const inboundSources = edges
+                .filter((edge) => edge.target === node.url)
+                .map((edge) => nodeByUrl.get(edge.source))
+                .filter((source) => Boolean(source));
+            if (inboundSources.length > 0 && inboundSources.every((source) => isLowValuePage(source))) {
+                orphanType = 'soft';
+            }
+        }
+        if (!orphanType) {
+            return { ...node, orphan: false };
+        }
+        if (!options.severityEnabled) {
+            return {
+                ...node,
+                orphan: true,
+                orphanType
+            };
+        }
+        const orphanSeverity = calculateOrphanSeverity(orphanType, { ...node, inLinks: inbound });
+        return {
+            ...node,
+            orphan: true,
+            orphanType,
+            orphanSeverity,
+            impactLevel: mapImpactLevel(orphanSeverity)
+        };
+    });
+}

package/dist/utils/version.d.ts

@@ -0,0 +1,2 @@
+declare let version: string;
+export { version };

package/dist/utils/version.js

@@ -0,0 +1,15 @@
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+let version = '0.0.1';
+try {
+    const pkgPath = join(__dirname, '../../package.json');
+    const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
+    version = pkg.version;
+}
+catch {
+    // Fallback to internal default
+}
+export { version };

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@crawlith/core",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "better-sqlite3": "^12.6.2",
+    "chalk": "^5.3.0",
+    "cheerio": "^1.0.0-rc.12",
+    "p-limit": "^5.0.0",
+    "robots-parser": "^3.0.1",
+    "undici": "^6.13.0",
+    "vite": "7.3.1"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^20.12.7",
+    "typescript": "^5.4.5",
+    "vitest": "^4.0.18"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run"
+  }
+}