@crawlith/core 0.1.0 → 0.1.2

package/dist/crawler/extract.js

@@ -2,8 +2,11 @@ import * as cheerio from 'cheerio';
 /**
  * Extracts all links from an HTML document.
  * Returns absolute URLs.
+ * @param html The HTML content string
+ * @param baseUrl The base URL to resolve relative links against
+ * @param onError Optional callback for handling extraction errors
  */
-export function extractLinks(html, baseUrl) {
+export function extractLinks(html, baseUrl, onError) {
     try {
         const $ = cheerio.load(html);
         const links = new Set();
@@ -27,7 +30,9 @@ export function extractLinks(html, baseUrl) {
         return Array.from(links);
     }
     catch (e) {
-        console.error(`Error extracting links from ${baseUrl}:`, e);
+        if (onError) {
+            onError(e);
+        }
         return [];
     }
 }

package/dist/crawler/fetcher.d.ts

@@ -23,9 +23,10 @@ export interface FetchOptions {
     crawlDelay?: number;
 }
 export declare class Fetcher {
-    private userAgent;
+    userAgent: string;
     private rateLimiter;
     private proxyAdapter;
+    private secureDispatcher;
     private scopeManager?;
     private maxRedirects;
     constructor(options?: {

package/dist/crawler/fetcher.js

@@ -1,26 +1,34 @@
 import { request } from 'undici';
+import * as net from 'net';
 import { IPGuard } from '../core/security/ipGuard.js';
 import { RateLimiter } from '../core/network/rateLimiter.js';
 import { RetryPolicy } from '../core/network/retryPolicy.js';
 import { ResponseLimiter } from '../core/network/responseLimiter.js';
 import { RedirectController } from '../core/network/redirectController.js';
 import { ProxyAdapter } from '../core/network/proxyAdapter.js';
-import { version } from '../utils/version.js';
+import { DEFAULTS } from '../constants.js';
 export class Fetcher {
-    userAgent = 'crawlith/1.0';
+    userAgent = DEFAULTS.USER_AGENT;
     rateLimiter;
     proxyAdapter;
+    secureDispatcher;
     scopeManager;
     maxRedirects;
     constructor(options = {}) {
-        this.rateLimiter = new RateLimiter(options.rate || 2);
+        this.rateLimiter = new RateLimiter(options.rate || DEFAULTS.RATE_LIMIT);
         this.proxyAdapter = new ProxyAdapter(options.proxyUrl);
+        if (this.proxyAdapter.dispatcher) {
+            this.secureDispatcher = this.proxyAdapter.dispatcher;
+        }
+        else {
+            this.secureDispatcher = IPGuard.getSecureDispatcher();
+        }
         this.scopeManager = options.scopeManager;
-        this.maxRedirects = Math.min(options.maxRedirects ?? 2, 11);
-        this.userAgent = options.userAgent || `crawlith/${version}`;
+        this.maxRedirects = Math.min(options.maxRedirects ?? DEFAULTS.MAX_REDIRECTS, DEFAULTS.MAX_REDIRECTS_LIMIT);
+        this.userAgent = options.userAgent || DEFAULTS.USER_AGENT;
     }
     async fetch(url, options = {}) {
-        const maxBytes = options.maxBytes || 2000000;
+        const maxBytes = options.maxBytes || DEFAULTS.MAX_BYTES;
         const redirectChain = [];
         const redirectController = new RedirectController(this.maxRedirects, url);
         let currentUrl = url;
@@ -28,10 +36,14 @@ export class Fetcher {
         // Use a while(true) and explicit return/continue to handle redirects
         while (true) {
             const urlObj = new URL(currentUrl);
-            // 1. SSRF Guard
-            const isSafe = await IPGuard.validateHost(urlObj.hostname);
-            if (!isSafe) {
-                return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries);
+            // 1. SSRF Guard (IP Literals only)
+            // We only check explicit IP literals here to fail fast.
+            // For domains, we rely on the secureDispatcher (which uses IPGuard.secureLookup)
+            // to resolve and validate the IP at connection time, preventing TOCTOU attacks.
+            if (net.isIP(urlObj.hostname)) {
+                if (IPGuard.isInternal(urlObj.hostname)) {
+                    return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries);
+                }
             }
             // 2. Scope Validation (Domain & Subdomain)
             if (this.scopeManager) {
@@ -61,7 +73,7 @@ export class Fetcher {
                         method: 'GET',
                         headers,
                         maxRedirections: 0,
-                        dispatcher: this.proxyAdapter.dispatcher,
+                        dispatcher: this.secureDispatcher,
                         headersTimeout: 10000,
                         bodyTimeout: 10000
                     });
@@ -141,6 +153,9 @@ export class Fetcher {
             catch (error) {
                 // Map common network errors to specific statuses if needed
                 const isProxyError = error.message?.toLowerCase().includes('proxy') || error.code === 'ECONNREFUSED';
+                if (error.code === 'EBLOCKED' || error.message?.includes('Blocked internal IP')) {
+                    return this.errorResult('blocked_internal_ip', currentUrl, redirectChain, totalRetries);
+                }
                 const finalStatus = isProxyError ? 'proxy_connection_failed' : 'network_error';
                 return this.errorResult(totalRetries >= RetryPolicy.DEFAULT_CONFIG.maxRetries ? 'failed_after_retries' : finalStatus, currentUrl, redirectChain, totalRetries);
             }

package/dist/crawler/metricsRunner.d.ts

@@ -1 +1,23 @@
-export declare function runPostCrawlMetrics(snapshotId: number, maxDepth: number, limitReached?: boolean): void;
+import { EngineContext } from '../events.js';
+import { Graph } from '../graph/graph.js';
+export interface PostCrawlOptions {
+    context?: EngineContext;
+    limitReached?: boolean;
+    graphInstance?: Graph;
+    clustering?: boolean;
+    clusterThreshold?: number;
+    minClusterSize?: number;
+    health?: boolean;
+    computePagerank?: boolean;
+    computeHits?: boolean;
+    heading?: boolean;
+    orphans?: boolean;
+    orphanSeverity?: 'low' | 'medium' | 'high' | boolean;
+    includeSoftOrphans?: boolean;
+    minInbound?: number;
+    rootOrigin?: string;
+}
+export declare function runPostCrawlMetrics(snapshotId: number, maxDepth: number, options?: PostCrawlOptions): {
+    metrics: any;
+    healthData?: any;
+} | undefined;

package/dist/crawler/metricsRunner.js

@@ -3,51 +3,189 @@ import { loadGraphFromSnapshot } from '../db/graphLoader.js';
 import { MetricsRepository } from '../db/repositories/MetricsRepository.js';
 import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
 import { PageRepository } from '../db/repositories/PageRepository.js';
-import { computePageRank } from '../graph/pagerank.js';
 import { calculateMetrics } from '../graph/metrics.js';
-import { computeHITS } from '../scoring/hits.js';
-export function runPostCrawlMetrics(snapshotId, maxDepth, limitReached = false) {
+import { PageRankService } from '../graph/pagerank.js';
+import { HITSService } from '../graph/hits.js';
+import { TrapDetector } from './trap.js';
+import { ClusteringService } from '../analysis/clustering.js';
+import { DuplicateService } from '../analysis/duplicate.js';
+import { annotateOrphans } from '../analysis/orphan.js';
+import { Soft404Service } from '../analysis/soft404.js';
+import { HeadingHealthService } from '../analysis/heading.js';
+import { HealthService } from '../scoring/health.js';
+import { analyzeContent } from '../analysis/content.js';
+import { load } from 'cheerio';
+export function runPostCrawlMetrics(snapshotId, maxDepth, options = {}) {
+    const context = options.context;
+    const limitReached = options.limitReached || false;
+    const graphInstance = options.graphInstance;
     const db = getDb();
     const metricsRepo = new MetricsRepository(db);
     const snapshotRepo = new SnapshotRepository(db);
     const pageRepo = new PageRepository(db);
+    const graph = graphInstance || loadGraphFromSnapshot(snapshotId);
+    // Fallback emitter
+    const emit = (event) => {
+        if (context) {
+            context.emit(event);
+        }
+        else {
+            if (event.type === 'error')
+                console.error(event.message);
+            else if (event.type !== 'debug') {
+                const out = event.message || event.phase;
+                if (out)
+                    console.log(out);
+            }
+        }
+    };
     const snapshot = snapshotRepo.getSnapshot(snapshotId);
     if (!snapshot) {
-        console.error(`Snapshot ${snapshotId} not found`);
-        return;
+        emit({ type: 'error', message: `Snapshot ${snapshotId} not found` });
+        return undefined;
     }
-    console.log('Loading graph for metrics calculation...');
-    const graph = loadGraphFromSnapshot(snapshotId);
-    console.log('Computing PageRank...');
-    computePageRank(graph);
-    console.log('Computing HITS...');
-    computeHITS(graph);
-    console.log('Updating metrics in DB...');
-    const nodes = graph.getNodes();
-    const tx = db.transaction(() => {
-        for (const node of nodes) {
-            const pageId = pageRepo.getIdByUrl(snapshot.site_id, node.url);
-            if (!pageId)
-                continue;
-            const existing = metricsRepo.getMetricsForPage(snapshotId, pageId);
-            metricsRepo.insertMetrics({
-                snapshot_id: snapshotId,
-                page_id: pageId,
-                authority_score: node.authorityScore ?? null,
-                hub_score: node.hubScore ?? null,
-                pagerank: node.pageRank ?? null,
-                pagerank_score: node.pageRankScore ?? null,
-                link_role: node.linkRole ?? null,
-                crawl_status: existing?.crawl_status ?? null,
-                word_count: existing?.word_count ?? null,
-                thin_content_score: existing?.thin_content_score ?? null,
-                external_link_ratio: existing?.external_link_ratio ?? null,
-                orphan_score: existing?.orphan_score ?? null,
-                duplicate_cluster_id: node.duplicateClusterId ?? null,
-                duplicate_type: node.duplicateType ?? null,
-                is_cluster_primary: node.isClusterPrimary ? 1 : 0
+    if (!graphInstance) {
+        emit({ type: 'metrics:start', phase: 'Loading graph' });
+    }
+    emit({ type: 'metrics:start', phase: 'Running core algorithms' });
+    // 1. Graph Algorithms
+    const prResults = options.computePagerank ? new PageRankService().evaluate(graph) : new Map();
+    const hitsResults = options.computeHits ? new HITSService().evaluate(graph, { iterations: 20 }) : new Map();
+    // 2. Crawler Safety
+    new TrapDetector().analyze(graph);
+    // 3. Analysis / Intelligence
+    if (options.clustering) {
+        const contentClusters = new ClusteringService().detectContentClusters(graph, options.clusterThreshold, options.minClusterSize);
+        if (contentClusters.length > 0) {
+            const insertCluster = db.prepare(`
+                INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
+                VALUES (@id, @snapshot_id, @count, @primary_url, @risk, @shared_path_prefix)
+            `);
+            const insertContentTx = db.transaction((clusters) => {
+                for (const c of clusters) {
+                    insertCluster.run({
+                        id: c.id,
+                        snapshot_id: snapshotId,
+                        count: c.count,
+                        primary_url: c.primaryUrl,
+                        risk: c.risk,
+                        shared_path_prefix: c.sharedPathPrefix ?? null
+                    });
+                }
             });
-            // Update page-level crawl trap data
+            insertContentTx(contentClusters);
+        }
+    }
+    const duplicateClusters = new DuplicateService().detectDuplicates(graph, { collapse: false });
+    if (duplicateClusters.length > 0) {
+        const insertCluster = db.prepare(`
+            INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
+            VALUES (@id, @snapshot_id, @type, @size, @representative, @severity)
+        `);
+        const insertDuplicateTx = db.transaction((clusters) => {
+            for (const c of clusters) {
+                insertCluster.run({
+                    id: c.id,
+                    snapshot_id: snapshotId,
+                    type: c.type, // valid: 'exact' | 'near' | 'template_heavy'
+                    size: c.size,
+                    representative: c.representative,
+                    severity: c.severity || 'low'
+                });
+            }
+        });
+        insertDuplicateTx(duplicateClusters);
+    }
+    let annotatedNodes = [];
+    if (options.orphans) {
+        const orphanOptions = {
+            enabled: true,
+            severityEnabled: !!options.orphanSeverity || options.orphanSeverity === undefined,
+            includeSoftOrphans: options.includeSoftOrphans ?? true,
+            minInbound: options.minInbound ?? 2
+        };
+        annotatedNodes = annotateOrphans(graph.getNodes(), graph.getEdges(), orphanOptions);
+    }
+    const soft404Service = new Soft404Service();
+    const headingService = new HeadingHealthService();
+    // Pre-calculate heading health for all nodes with HTML
+    let headingPayloads = new Map();
+    if (options.heading) {
+        const result = headingService.evaluateNodes(graph.getNodes());
+        headingPayloads = result.payloadsByUrl;
+    }
+    // Apply signals to nodes
+    for (const node of graph.getNodes()) {
+        const pr = prResults.get(node.url);
+        if (pr)
+            node.pagerankScore = pr.score;
+        const hits = hitsResults.get(node.url);
+        if (hits) {
+            node.authScore = hits.authority_score;
+            node.hubScore = hits.hub_score;
+            node.linkRole = hits.link_role;
+        }
+        if (options.orphans) {
+            const annotated = annotatedNodes.find((n) => n.url === node.url);
+            if (annotated) {
+                node.orphanScore = annotated.orphanSeverity;
+                node.orphanType = annotated.orphanType;
+                node.impactLevel = annotated.impactLevel;
+            }
+        }
+        if (options.heading) {
+            const heading = headingPayloads.get(node.url);
+            if (heading) {
+                node.headingScore = heading.score;
+                node.headingData = JSON.stringify(heading);
+            }
+        }
+        if (node.html) {
+            const soft404 = soft404Service.analyze(node.html, node.outLinks);
+            node.soft404Score = soft404.score;
+            const $ = load(node.html);
+            const content = analyzeContent($);
+            node.wordCount = content.wordCount;
+        }
+    }
+    emit({ type: 'metrics:start', phase: 'Updating metrics in DB' });
+    // Pre-fetch all page IDs to avoid N+1 queries
+    const pagesIdentity = pageRepo.getPagesIdentityBySnapshot(snapshotId);
+    const urlToId = new Map();
+    for (const p of pagesIdentity) {
+        urlToId.set(p.normalized_url, p.id);
+    }
+    const metricsToSave = graph.getNodes().map(node => {
+        const pageId = urlToId.get(node.url);
+        if (!pageId)
+            return null;
+        return {
+            snapshot_id: snapshotId,
+            page_id: pageId,
+            crawl_status: node.crawlStatus ?? null,
+            word_count: node.wordCount ?? null,
+            thin_content_score: node.thinContentScore ?? null,
+            external_link_ratio: node.externalLinkRatio ?? null,
+            pagerank_score: node.pagerankScore ?? null,
+            hub_score: node.hubScore ?? null,
+            auth_score: node.authScore ?? null,
+            link_role: node.linkRole ?? null,
+            duplicate_cluster_id: node.duplicateClusterId ?? null,
+            duplicate_type: node.duplicateType ?? null,
+            cluster_id: node.clusterId ?? null,
+            soft404_score: node.soft404Score ?? null,
+            heading_score: node.headingScore ?? null,
+            orphan_score: node.orphanScore ?? null,
+            orphan_type: node.orphanType ?? null,
+            impact_level: node.impactLevel ?? null,
+            heading_data: node.headingData ?? null,
+            is_cluster_primary: node.isClusterPrimary ? 1 : 0
+        };
+    }).filter(m => m !== null);
+    metricsRepo.insertMany(metricsToSave);
+    // Update page-level metadata in transaction
+    const tx = db.transaction(() => {
+        for (const node of graph.getNodes()) {
             if (node.crawlTrapFlag || node.redirectChain?.length || node.bytesReceived) {
                 pageRepo.upsertPage({
                     site_id: snapshot.site_id,
@@ -61,48 +199,40 @@ export function runPostCrawlMetrics(snapshotId, maxDepth, limitReached = false)
                 });
             }
         }
-        // Save duplicate clusters
-        if (graph.duplicateClusters.length > 0) {
-            const clusterStmt = db.prepare(`
-                INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
-                VALUES (?, ?, ?, ?, ?, ?)
-            `);
-            for (const cluster of graph.duplicateClusters) {
-                clusterStmt.run(cluster.id, snapshotId, cluster.type, cluster.size, cluster.representative, cluster.severity);
-            }
-        }
-        // Save content clusters
-        if (graph.contentClusters.length > 0) {
-            const contentStmt = db.prepare(`
-                INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
-                VALUES (?, ?, ?, ?, ?, ?)
-            `);
-            for (const cluster of graph.contentClusters) {
-                contentStmt.run(cluster.id, snapshotId, cluster.count, cluster.primaryUrl, cluster.risk, cluster.sharedPathPrefix ?? null);
-            }
-        }
     });
     tx();
-    console.log('Computing aggregate stats...');
+    emit({ type: 'metrics:start', phase: 'Computing aggregate stats' });
     const metrics = calculateMetrics(graph, maxDepth);
-    let totalScore = 0;
-    let totalWeight = 0;
-    for (const node of nodes) {
-        const score = node.authorityScore || node.pageRankScore || 0;
-        const depth = node.depth;
-        const weight = 1 / (depth + 1);
-        totalScore += score * weight;
-        totalWeight += weight;
+    // Compute health score if enabled
+    let healthScore = null;
+    if (options.health) {
+        try {
+            const rootOrigin = options.rootOrigin ?? '';
+            const healthService = new HealthService();
+            const issues = healthService.collectCrawlIssues(graph, metrics, rootOrigin);
+            const breakdown = healthService.calculateHealthScore(metrics.totalPages, issues);
+            healthScore = breakdown.score;
+        }
+        catch (e) {
+            emit({ type: 'error', message: 'Error computing health score', error: e });
+        }
     }
-    const healthScore = totalWeight > 0 ? (totalScore / totalWeight) * 100 : 0;
-    const thinCountRow = db.prepare('SELECT count(*) as count FROM metrics WHERE snapshot_id = ? AND thin_content_score >= 70').get(snapshotId);
+    const thinContentCount = graph.getNodes().filter(n => n.wordCount !== undefined && n.wordCount < 200 && n.status === 200).length;
+    const orphanCount = metrics.orphanPages.length;
     snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
         node_count: metrics.totalPages,
         edge_count: metrics.totalEdges,
-        health_score: healthScore,
-        orphan_count: metrics.orphanPages.length,
-        thin_content_count: thinCountRow.count,
-        limit_reached: limitReached ? 1 : 0
+        limit_reached: limitReached ? 1 : 0,
+        thin_content_count: thinContentCount,
+        orphan_count: orphanCount,
+        ...(healthScore !== null ? { health_score: healthScore } : {})
     });
-    console.log('Metrics calculation complete.');
+    emit({ type: 'metrics:complete', durationMs: 0 });
+    return {
+        metrics,
+        healthData: healthScore !== null ? {
+            health: new HealthService().calculateHealthScore(metrics.totalPages, new HealthService().collectCrawlIssues(graph, metrics, options.rootOrigin ?? '')),
+            issues: new HealthService().collectCrawlIssues(graph, metrics, options.rootOrigin ?? '')
+        } : undefined
+    };
 }

package/dist/crawler/normalize.d.ts

@@ -3,5 +3,46 @@
  */
 export interface NormalizeOptions {
     stripQuery?: boolean;
+    toPath?: boolean;
 }
 export declare function normalizeUrl(input: string, base: string, options?: NormalizeOptions): string | null;
+/**
+ * Utility for converting between absolute URLs and relative paths
+ * primarily used for database storage.
+ */
+export declare class UrlUtil {
+    /**
+     * Extract a stable domain key from a URL/domain input.
+     * Examples:
+     *  - "https://www.example.com/a" -> "example.com"
+     *  - "example.com" -> "example.com"
+     */
+    static extractDomain(input: string): string;
+    /**
+     * Resolve a site's absolute origin from persisted site fields.
+     */
+    static resolveSiteOrigin(site: {
+        domain: string;
+        preferred_url?: string | null;
+        ssl?: number | null;
+    }): string;
+    /**
+     * Converts a full URL to a root-relative path if it matches the origin.
+     * If it doesn't match the origin, it's considered external and kept absolute.
+     */
+    static toPath(urlStr: string, origin: string): string;
+    /**
+     * Converts a root-relative path back to an absolute URL relative to the origin.
+     * If the input is already an absolute URL, it is returned as-is.
+     */
+    static toAbsolute(pathOrUrl: string, origin: string): string;
+    /**
+     * Determines if a URL (or path) is internal relative to the origin.
+     */
+    static isInternal(pathOrUrl: string, origin: string): boolean;
+    /**
+     * Build normalized lookup candidates for querying pages table.
+     * Returns path/absolute/original variants in priority order, deduplicated.
+     */
+    static toLookupCandidates(input: string, origin: string): string[];
+}

package/dist/crawler/normalize.js

@@ -10,7 +10,7 @@ const TRACKING_PARAMS = new Set([
 ]);
 const SKIP_EXTENSIONS = new Set([
     '.pdf', '.jpg', '.png', '.svg', '.webp', '.gif',
-    '.zip', '.xml', '.json', '.mp4'
+    '.zip', '.xml', '.json', '.mp4', '.avif', '.ics'
 ]);
 export function normalizeUrl(input, base, options = {}) {
     try {
@@ -71,6 +71,7 @@ export function normalizeUrl(input, base, options = {}) {
             pathname = pathname.slice(0, -1);
         }
         u.pathname = pathname;
+        const finalUrl = u.toString();
         // 9. Skip non-HTML assets by extension
         const lastDotIndex = u.pathname.lastIndexOf('.');
         if (lastDotIndex !== -1) {
@@ -79,10 +80,125 @@ export function normalizeUrl(input, base, options = {}) {
                 return null;
             }
         }
-        // 10. Return final string
-        return u.toString();
+        // 10. Return path if requested
+        if (options.toPath) {
+            return u.pathname + u.search;
+        }
+        // 11. Return final string
+        return finalUrl;
     }
     catch (_e) {
         return null;
     }
 }
+/**
+ * Utility for converting between absolute URLs and relative paths
+ * primarily used for database storage.
+ */
+export class UrlUtil {
+    /**
+     * Extract a stable domain key from a URL/domain input.
+     * Examples:
+     *  - "https://www.example.com/a" -> "example.com"
+     *  - "example.com" -> "example.com"
+     */
+    static extractDomain(input) {
+        const trimmed = input.trim();
+        if (!trimmed)
+            return '';
+        try {
+            const direct = new URL(trimmed);
+            return direct.hostname.toLowerCase().replace(/^www\./, '');
+        }
+        catch {
+            // fall through
+        }
+        try {
+            const withProtocol = new URL(`https://${trimmed}`);
+            return withProtocol.hostname.toLowerCase().replace(/^www\./, '');
+        }
+        catch {
+            return trimmed.toLowerCase().replace(/^www\./, '');
+        }
+    }
+    /**
+     * Resolve a site's absolute origin from persisted site fields.
+     */
+    static resolveSiteOrigin(site) {
+        if (site.preferred_url) {
+            try {
+                return new URL(site.preferred_url).origin;
+            }
+            catch {
+                // fall through to domain+ssl fallback
+            }
+        }
+        const protocol = site.ssl === 0 ? 'http' : 'https';
+        return `${protocol}://${site.domain}`;
+    }
+    /**
+     * Converts a full URL to a root-relative path if it matches the origin.
+     * If it doesn't match the origin, it's considered external and kept absolute.
+     */
+    static toPath(urlStr, origin) {
+        try {
+            const url = new URL(urlStr);
+            const originUrl = new URL(origin);
+            if (url.origin === originUrl.origin) {
+                return url.pathname + url.search;
+            }
+            return urlStr;
+        }
+        catch {
+            return urlStr;
+        }
+    }
+    /**
+     * Converts a root-relative path back to an absolute URL relative to the origin.
+     * If the input is already an absolute URL, it is returned as-is.
+     */
+    static toAbsolute(pathOrUrl, origin) {
+        if (pathOrUrl.startsWith('http://') || pathOrUrl.startsWith('https://')) {
+            return pathOrUrl;
+        }
+        try {
+            return new URL(pathOrUrl, origin).toString();
+        }
+        catch {
+            return pathOrUrl;
+        }
+    }
+    /**
+     * Determines if a URL (or path) is internal relative to the origin.
+     */
+    static isInternal(pathOrUrl, origin) {
+        if (!pathOrUrl.startsWith('http'))
+            return true;
+        try {
+            const url = new URL(pathOrUrl);
+            const originUrl = new URL(origin);
+            return url.origin === originUrl.origin;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Build normalized lookup candidates for querying pages table.
+     * Returns path/absolute/original variants in priority order, deduplicated.
+     */
+    static toLookupCandidates(input, origin) {
+        const candidates = new Set();
+        const raw = input.trim();
+        if (!raw)
+            return [];
+        const absolute = normalizeUrl(raw, origin, { stripQuery: false }) || UrlUtil.toAbsolute(raw, origin);
+        const path = normalizeUrl(raw, origin, { stripQuery: false, toPath: true }) || UrlUtil.toPath(raw, origin);
+        const absolutePath = normalizeUrl(absolute, '', { stripQuery: false, toPath: true }) || UrlUtil.toPath(absolute, origin);
+        candidates.add(path);
+        candidates.add(absolute);
+        candidates.add(absolutePath);
+        candidates.add(raw);
+        return Array.from(candidates).filter(Boolean);
+    }
+}

package/dist/crawler/parser.d.ts

@@ -11,12 +11,10 @@ export interface ParseResult {
     contentHash: string;
     simhash?: string;
     uniqueTokenRatio?: number;
-    soft404Score: number;
-    soft404Signals: string[];
 }
 export declare class Parser {
     /**
      * Parses HTML content to extract metadata and links.
      */
-    parse(html: string, baseUrl: string, status: number): ParseResult;
+    parse(html: string, baseUrl: string, _status: number): ParseResult;
 }