@crawlith/core 0.1.0 → 0.1.2

package/dist/crawler/crawler.js

@@ -0,0 +1,683 @@
+import chalk from '../utils/chalk.js';
+import pLimit from 'p-limit';
+import robotsParser from 'robots-parser';
+import { Fetcher } from './fetcher.js';
+import { Parser } from './parser.js';
+import { Sitemap } from './sitemap.js';
+import { normalizeUrl, UrlUtil } from './normalize.js';
+import { UrlResolver } from './resolver.js';
+import { ScopeManager } from '../core/scope/scopeManager.js';
+import { getDb } from '../db/index.js';
+import { SiteRepository } from '../db/repositories/SiteRepository.js';
+import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
+import { PageRepository } from '../db/repositories/PageRepository.js';
+import { EdgeRepository } from '../db/repositories/EdgeRepository.js';
+import { MetricsRepository } from '../db/repositories/MetricsRepository.js';
+import { analyzeContent, calculateThinContentScore } from '../analysis/content.js';
+import { analyzeLinks } from '../analysis/links.js';
+import { DEFAULTS } from '../constants.js';
+// Fallback context for backward compatibility or when no context is provided
+const nullContext = {
+    emit: (event) => {
+        // Basic console fallback for critical events if no listener is attached
+        // This maintains some visibility for consumers not using the event system
+        if (event.type === 'error') {
+            console.error(event.message, event.error || '');
+        }
+        else if (event.type === 'warn') {
+            console.warn(event.message);
+        }
+    }
+};
+export class Crawler {
+    startUrl;
+    options;
+    context;
+    registry;
+    visited;
+    uniqueQueue;
+    queue;
+    active;
+    pagesCrawled;
+    reachedLimit;
+    maxDepthInCrawl;
+    concurrency;
+    limitConcurrency;
+    // Repositories
+    siteRepo = null;
+    snapshotRepo = null;
+    pageRepo = null;
+    edgeRepo = null;
+    metricsRepo = null;
+    // Site/Snapshot info
+    siteId = null;
+    snapshotId = null;
+    reusingSnapshot = false;
+    runType = 'completed';
+    rootOrigin = '';
+    // Discovery tracking
+    discoveryDepths = new Map();
+    // Buffers for batch operations
+    pageBuffer = new Map();
+    edgeBuffer = [];
+    metricsBuffer = [];
+    pendingSitemaps = 0;
+    edgesFound = 0;
+    lastProgressEmitAt = 0;
+    progressPhase = 'crawling';
+    // Modules
+    scopeManager = null;
+    fetcher = null;
+    parser = null;
+    sitemapFetcher = null;
+    robots = null;
+    constructor(startUrl, options, context) {
+        this.startUrl = startUrl;
+        this.options = options;
+        this.context = context || nullContext;
+        this.registry = options.registry;
+        this.visited = new Set();
+        this.uniqueQueue = new Set();
+        this.queue = [];
+        this.active = 0;
+        this.pagesCrawled = 0;
+        this.reachedLimit = false;
+        this.maxDepthInCrawl = Math.min(options.depth || DEFAULTS.MAX_DEPTH, DEFAULTS.MAX_DEPTH_LIMIT);
+        this.concurrency = Math.min(options.concurrency || DEFAULTS.CONCURRENCY, DEFAULTS.CONCURRENCY_LIMIT);
+        this.limitConcurrency = pLimit(this.concurrency);
+    }
+    toStorageUrl(url) {
+        return UrlUtil.isInternal(url, this.rootOrigin) ? UrlUtil.toPath(url, this.rootOrigin) : url;
+    }
+    async initialize() {
+        const db = getDb();
+        this.siteRepo = new SiteRepository(db);
+        this.snapshotRepo = new SnapshotRepository(db);
+        this.pageRepo = new PageRepository(db);
+        this.edgeRepo = new EdgeRepository(db);
+        this.metricsRepo = new MetricsRepository(db);
+        // Use resolver to find canonical origin and SSL
+        const resolver = new UrlResolver();
+        const tempFetcher = new Fetcher({ userAgent: this.options.userAgent, rate: this.options.rate });
+        const resolved = await resolver.resolve(this.startUrl, tempFetcher);
+        this.rootOrigin = resolved.url;
+        // Use the resolved absolute URL as the base — NOT this.startUrl which may be
+        // a bare domain (e.g. 'callforpaper.org') that would be treated as a relative
+        // path when passed to normalizeUrl, producing '/callforpaper.org'.
+        const rootUrl = normalizeUrl(this.rootOrigin, '', { stripQuery: this.options.stripQuery });
+        if (!rootUrl)
+            throw new Error('Invalid start URL');
+        const urlObj = new URL(this.rootOrigin);
+        const domain = urlObj.hostname.replace('www.', '');
+        const site = this.siteRepo.firstOrCreateSite(domain);
+        this.siteId = site.id;
+        // Persist the resolved preferred URL and SSL status
+        this.siteRepo.updateSitePreference(this.siteId, {
+            preferred_url: this.rootOrigin,
+            ssl: this.rootOrigin.startsWith('https') ? 1 : 0
+        });
+        this.rootOrigin = urlObj.origin;
+        // Keep storage path-first for internal URLs and reconcile any legacy absolute rows.
+        this.pageRepo.reconcileInternalUrls(this.siteId, this.rootOrigin);
+        this.startUrl = this.toStorageUrl(rootUrl);
+        // Now that rootOrigin is resolved, initialize ScopeManager with the correct absolute origin
+        this.scopeManager = new ScopeManager({
+            allowedDomains: this.options.allowedDomains || [],
+            deniedDomains: this.options.deniedDomains || [],
+            includeSubdomains: this.options.includeSubdomains || false,
+            rootUrl: this.rootOrigin
+        });
+        // Update fetcher with the now-initialized scopeManager
+        if (this.fetcher) {
+            this.fetcher.scopeManager = this.scopeManager;
+        }
+        // Every scan now creates a new snapshot (no reuse)
+        const runType = this.options.snapshotRunType || (this.options.previousGraph ? 'incremental' : 'completed');
+        this.snapshotId = this.snapshotRepo.createSnapshot(this.siteId, runType);
+        this.runType = runType;
+        // Expose snapshot context for plugins that persist per-snapshot data.
+        this.context.snapshotId = this.snapshotId;
+        // Seed discovery depth for root
+        this.discoveryDepths.set(this.startUrl, 0);
+    }
+    setupModules() {
+        this.fetcher = new Fetcher({
+            rate: this.options.rate,
+            proxyUrl: this.options.proxyUrl,
+            scopeManager: this.scopeManager ?? undefined,
+            maxRedirects: this.options.maxRedirects,
+            userAgent: this.options.userAgent
+        });
+        this.parser = new Parser();
+        this.sitemapFetcher = new Sitemap(this.context, this.fetcher);
+    }
+    async fetchRobots() {
+        const robotsUrl = new URL('/robots.txt', this.rootOrigin).toString();
+        try {
+            const res = await this.fetcher.fetch(robotsUrl, { maxBytes: 500000 });
+            if (res && typeof res.status === 'number' && res.status >= 200 && res.status < 300) {
+                this.robots = robotsParser(robotsUrl, res.body);
+            }
+        }
+        catch {
+            // Suppressed expected network warnings when robots block
+            console.warn('Failed to fetch robots.txt, proceeding...');
+        }
+    }
+    shouldEnqueue(url, depth) {
+        if (this.visited.has(url))
+            return false;
+        if (this.uniqueQueue.has(url))
+            return false;
+        if (depth > this.maxDepthInCrawl)
+            return false;
+        if (this.scopeManager.isUrlEligible(url) !== 'allowed')
+            return false;
+        if (this.registry) {
+            const allowed = this.registry.runSyncBailHook('shouldEnqueueUrl', this.context, url, depth);
+            if (allowed === false)
+                return false;
+        }
+        return true;
+    }
+    addToQueue(u, d, data = {}) {
+        if (this.scopeManager.isUrlEligible(u) !== 'allowed')
+            return;
+        if (!this.uniqueQueue.has(u)) {
+            this.uniqueQueue.add(u);
+            this.queue.push({ url: u, depth: d });
+            this.context.emit({ type: 'queue:enqueue', url: u, depth: d });
+            this.emitProgress();
+            this.bufferPage(u, d, 0, data);
+            const currentDiscovery = this.discoveryDepths.get(u);
+            if (currentDiscovery === undefined || d < currentDiscovery) {
+                this.discoveryDepths.set(u, d);
+            }
+        }
+    }
+    async seedQueue() {
+        // Seed from startUrl first to ensure it's prioritized in the queue
+        this.addToQueue(this.startUrl, 0);
+        const sitemapsToFetch = new Set();
+        // 1. Explicitly configured sitemap
+        if (this.options.sitemap && this.runType !== 'single') {
+            const explicitUrl = this.options.sitemap === 'true' || this.options.sitemap === true
+                ? new URL('/sitemap.xml', this.rootOrigin).toString()
+                : this.options.sitemap;
+            if (typeof explicitUrl === 'string' && explicitUrl.startsWith('http')) {
+                sitemapsToFetch.add(explicitUrl);
+            }
+        }
+        // 2. Discover sitemaps from robots.txt (unless explicitly disabled)
+        // Only auto-fetch on the FIRST real crawl (full/incremental).
+        // page --live reuses snapshots and should NOT trigger sitemap fetch.
+        const isFirstFullCrawl = this.runType !== 'single' && !this.snapshotRepo?.hasFullCrawl(this.siteId);
+        if (this.options.sitemap !== false && (this.options.sitemap || isFirstFullCrawl) && this.robots && this.runType !== 'single') {
+            const robotsSitemaps = this.robots.getSitemaps();
+            for (const s of robotsSitemaps) {
+                if (s)
+                    sitemapsToFetch.add(s);
+            }
+        }
+        // Process all discovered sitemaps in background
+        if (sitemapsToFetch.size > 0) {
+            for (const sitemapUrl of sitemapsToFetch) {
+                this.pendingSitemaps++;
+                // KICK OFF BACKGROUND TASK (Un-awaited)
+                (async () => {
+                    try {
+                        this.context.emit({ type: 'debug', message: 'Fetching sitemap in background', context: { url: sitemapUrl } });
+                        const sitemapUrls = await this.sitemapFetcher.fetch(sitemapUrl);
+                        if (sitemapUrls.length > 0) {
+                            this.context.emit({ type: 'debug', message: `Mapping ${sitemapUrls.length} URLs from sitemap... (Background)` });
+                            const sitemapEntries = sitemapUrls.map(u => {
+                                const normalized = normalizeUrl(u, this.rootOrigin, this.options);
+                                if (!normalized)
+                                    return null;
+                                const path = this.toStorageUrl(normalized);
+                                return {
+                                    site_id: this.siteId,
+                                    normalized_url: path,
+                                    first_seen_snapshot_id: this.snapshotId,
+                                    last_seen_snapshot_id: this.snapshotId,
+                                    discovered_via_sitemap: 1,
+                                    depth: 0,
+                                    http_status: 0
+                                };
+                            }).filter((p) => p !== null);
+                            // Bulk register to DB
+                            this.pageRepo.upsertMany(sitemapEntries);
+                            // Add to queue for Actual Crawling
+                            for (const entry of sitemapEntries) {
+                                this.addToQueue(entry.normalized_url, 0, { discovered_via_sitemap: 1 });
+                            }
+                        }
+                    }
+                    catch (e) {
+                        this.context.emit({ type: 'warn', message: 'Sitemap fetch failed', context: { url: sitemapUrl, error: String(e) } });
+                    }
+                    finally {
+                        this.pendingSitemaps--;
+                    }
+                })();
+            }
+        }
+    }
+    bufferPage(url, depth, status, data = {}) {
+        const existing = this.pageBuffer.get(url);
+        const knownDiscovery = this.discoveryDepths.get(url);
+        // Always use the best (minimum) depth discovered for this URL
+        const finalDepth = knownDiscovery !== undefined ? Math.min(knownDiscovery, depth) : depth;
+        if (knownDiscovery === undefined || depth < knownDiscovery) {
+            this.discoveryDepths.set(url, depth);
+        }
+        // If we already have a buffered record, only update if the new one is more "complete" (has status)
+        // or if the depth is better.
+        if (existing) {
+            const isStatusUpdate = status !== 0 && existing.http_status === 0;
+            const isBetterDepth = finalDepth < existing.depth;
+            if (!isStatusUpdate && !isBetterDepth && Object.keys(data).length === 0) {
+                return;
+            }
+            this.pageBuffer.set(url, {
+                ...existing,
+                depth: finalDepth,
+                http_status: status !== 0 ? status : existing.http_status,
+                ...data
+            });
+        }
+        else {
+            this.pageBuffer.set(url, {
+                site_id: this.siteId,
+                normalized_url: url,
+                depth: finalDepth,
+                http_status: status,
+                last_seen_snapshot_id: this.snapshotId,
+                ...data
+            });
+        }
+        if (this.pageBuffer.size >= 50) {
+            this.flushPages();
+        }
+    }
+    flushPages() {
+        if (this.pageBuffer.size === 0)
+            return;
+        this.pageRepo.upsertMany(Array.from(this.pageBuffer.values()));
+        this.pageBuffer.clear();
+    }
+    bufferEdge(sourceUrl, targetUrl, weight = 1.0, rel = 'internal') {
+        this.edgeBuffer.push({ sourceUrl, targetUrl, weight, rel });
+        this.edgesFound += 1;
+        this.emitProgress();
+        if (this.edgeBuffer.length >= 100) {
+            this.flushEdges();
+        }
+    }
+    emitProgress(force = false) {
+        const now = Date.now();
+        if (!force && now - this.lastProgressEmitAt < 200)
+            return;
+        this.lastProgressEmitAt = now;
+        this.context.emit({
+            type: 'crawl:progress',
+            pagesCrawled: this.pagesCrawled,
+            queued: this.queue.length,
+            active: this.active,
+            nodesFound: this.uniqueQueue.size,
+            edgesFound: this.edgesFound,
+            phase: this.progressPhase
+        });
+    }
+    flushEdges() {
+        if (this.edgeBuffer.length === 0)
+            return;
+        // To resolve URLs to IDs, we need to make sure pages are flushed first
+        this.flushPages();
+        const identities = this.pageRepo.getPagesIdentityBySnapshot(this.snapshotId);
+        const urlToId = new Map(identities.map(p => [p.normalized_url, p.id]));
+        // When reusing a snapshot, clean up stale edges for pages being re-crawled
+        if (this.reusingSnapshot) {
+            const sourcePageIds = new Set(this.edgeBuffer.map(e => urlToId.get(e.sourceUrl)).filter((id) => id !== undefined));
+            for (const pageId of sourcePageIds) {
+                this.edgeRepo.deleteEdgesForPage(this.snapshotId, pageId);
+            }
+        }
+        const edgesToInsert = this.edgeBuffer
+            .map(e => ({
+            snapshot_id: this.snapshotId,
+            source_page_id: urlToId.get(e.sourceUrl),
+            target_page_id: urlToId.get(e.targetUrl),
+            weight: e.weight,
+            rel: e.rel
+        }))
+            .filter(e => e.source_page_id !== undefined && e.target_page_id !== undefined);
+        if (edgesToInsert.length > 0) {
+            this.edgeRepo.insertEdges(edgesToInsert);
+        }
+        this.edgeBuffer = [];
+    }
+    bufferMetrics(url, data) {
+        this.metricsBuffer.push({ url, data });
+        if (this.metricsBuffer.length >= 50) {
+            this.flushMetrics();
+        }
+    }
+    flushMetrics() {
+        if (this.metricsBuffer.length === 0)
+            return;
+        this.flushPages();
+        const identities = this.pageRepo.getPagesIdentityBySnapshot(this.snapshotId);
+        const urlToId = new Map(identities.map(p => [p.normalized_url, p.id]));
+        const metricsList = this.metricsBuffer.map(item => {
+            const pageId = urlToId.get(item.url);
+            if (!pageId)
+                return null;
+            return {
+                snapshot_id: this.snapshotId,
+                page_id: pageId,
+                crawl_status: null,
+                word_count: null,
+                thin_content_score: null,
+                external_link_ratio: null,
+                pagerank_score: null,
+                hub_score: null,
+                auth_score: null,
+                link_role: null,
+                duplicate_cluster_id: null,
+                duplicate_type: null,
+                cluster_id: null,
+                soft404_score: null,
+                heading_score: null,
+                orphan_score: null,
+                orphan_type: null,
+                impact_level: null,
+                heading_data: null,
+                is_cluster_primary: 0,
+                ...item.data
+            };
+        }).filter(m => m !== null);
+        if (metricsList.length > 0) {
+            this.metricsRepo.insertMany(metricsList);
+        }
+        this.metricsBuffer = [];
+    }
+    async flushAll() {
+        this.flushPages();
+        this.flushEdges();
+        this.flushMetrics();
+    }
+    async fetchPage(url, depth, prevNode) {
+        const startTime = Date.now();
+        try {
+            this.context.emit({ type: 'crawl:start', url });
+            const res = await this.fetcher.fetch(url, {
+                maxBytes: this.options.maxBytes,
+                crawlDelay: this.robots ? this.robots.getCrawlDelay('crawlith') : undefined,
+                etag: prevNode?.etag,
+                lastModified: prevNode?.lastModified
+            });
+            const durationMs = Date.now() - startTime;
+            this.context.emit({
+                type: 'crawl:success',
+                url,
+                status: typeof res.status === 'number' ? res.status : 0,
+                durationMs,
+                depth
+            });
+            return res;
+        }
+        catch (e) {
+            this.context.emit({ type: 'crawl:error', url, error: String(e), depth });
+            return null;
+        }
+    }
+    handleCachedResponse(url, finalUrl, depth, prevNode) {
+        const path = url;
+        const finalPath = this.toStorageUrl(finalUrl);
+        this.bufferPage(finalPath, depth, prevNode.status, {
+            html: prevNode.html,
+            canonical_url: prevNode.canonical,
+            noindex: prevNode.noindex ? 1 : 0,
+            nofollow: prevNode.nofollow ? 1 : 0,
+            content_hash: prevNode.contentHash,
+            simhash: prevNode.simhash,
+            etag: prevNode.etag,
+            last_modified: prevNode.lastModified
+        });
+        this.bufferMetrics(finalPath, {
+            crawl_status: 'cached',
+            word_count: prevNode.wordCount,
+            thin_content_score: prevNode.thinContentScore,
+            external_link_ratio: prevNode.externalLinkRatio
+        });
+        // Re-discovery links from previous graph to continue crawling if needed
+        const prevLinks = this.options.previousGraph?.getEdges()
+            .filter(e => e.source === path)
+            .map(e => e.target);
+        if (prevLinks) {
+            for (const link of prevLinks) {
+                const normalizedLink = normalizeUrl(link, this.rootOrigin, this.options);
+                if (normalizedLink) {
+                    const path = this.toStorageUrl(normalizedLink);
+                    if (path !== url) {
+                        this.bufferPage(path, depth + 1, 0);
+                        this.bufferEdge(url, path, 1.0, 'internal');
+                        if (this.shouldEnqueue(path, depth + 1)) {
+                            this.addToQueue(path, depth + 1);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    handleRedirects(chain, depth) {
+        for (const step of chain) {
+            const sourceAbs = normalizeUrl(step.url, this.rootOrigin, this.options);
+            const targetAbs = normalizeUrl(step.target, this.rootOrigin, this.options);
+            if (sourceAbs && targetAbs) {
+                const sourcePath = this.toStorageUrl(sourceAbs);
+                const targetPath = this.toStorageUrl(targetAbs);
+                const sourceInternal = UrlUtil.isInternal(sourceAbs, this.rootOrigin);
+                const targetInternal = UrlUtil.isInternal(targetAbs, this.rootOrigin);
+                this.bufferPage(sourcePath, depth, step.status, { is_internal: sourceInternal ? 1 : 0 });
+                this.bufferPage(targetPath, depth, 0, { is_internal: targetInternal ? 1 : 0 });
+                this.bufferEdge(sourcePath, targetPath, 1.0, targetInternal ? 'internal' : 'external');
+            }
+        }
+    }
+    handleSuccessResponse(res, path, absoluteUrl, depth, isBlocked = false) {
+        const contentTypeHeader = res.headers['content-type'];
+        const contentType = Array.isArray(contentTypeHeader) ? contentTypeHeader[0] : (contentTypeHeader || '');
+        if (!contentType || !contentType.toLowerCase().includes('text/html')) {
+            this.bufferPage(path, depth, typeof res.status === 'number' ? res.status : 0);
+            return;
+        }
+        const parseResult = this.parser.parse(res.body, absoluteUrl, res.status);
+        if (this.registry) {
+            this.registry.runHook('onPageParsed', this.context, {
+                url: absoluteUrl,
+                status: res.status,
+                depth: depth,
+                headers: res.headers,
+                ...parseResult
+            });
+        }
+        this.bufferPage(path, depth, res.status, {
+            html: parseResult.html,
+            canonical_url: parseResult.canonical || undefined,
+            noindex: parseResult.noindex ? 1 : 0,
+            nofollow: parseResult.nofollow ? 1 : 0,
+            content_hash: parseResult.contentHash,
+            simhash: parseResult.simhash,
+            etag: res.etag,
+            last_modified: res.lastModified,
+            retries: res.retries,
+            bytes_received: res.bytesReceived
+        });
+        try {
+            const contentAnalysis = analyzeContent(parseResult.html);
+            const linkAnalysis = analyzeLinks(parseResult.html, absoluteUrl, this.rootOrigin);
+            const thinScore = calculateThinContentScore(contentAnalysis, 0);
+            this.bufferMetrics(path, {
+                crawl_status: isBlocked ? 'blocked_by_robots' : 'fetched',
+                word_count: contentAnalysis.wordCount,
+                thin_content_score: thinScore,
+                external_link_ratio: linkAnalysis.externalRatio
+            });
+        }
+        catch (e) {
+            this.context.emit({ type: 'error', message: 'Error calculating per-page metrics', error: e, context: { url: absoluteUrl } });
+        }
+        for (const linkItem of parseResult.links) {
+            const normalizedLink = normalizeUrl(linkItem.url, absoluteUrl, this.options);
+            if (normalizedLink) {
+                const targetPath = this.toStorageUrl(normalizedLink);
+                if (targetPath !== path) {
+                    const isInternal = UrlUtil.isInternal(normalizedLink, this.rootOrigin);
+                    this.bufferPage(targetPath, depth + 1, 0, { is_internal: isInternal ? 1 : 0 });
+                    this.bufferEdge(path, targetPath, 1.0, isInternal ? 'internal' : 'external');
+                    if (isInternal && this.shouldEnqueue(targetPath, depth + 1)) {
+                        this.addToQueue(targetPath, depth + 1);
+                    }
+                }
+            }
+        }
+    }
+    async processPage(item, isBlocked = false) {
+        const { url, depth } = item;
+        if (this.scopeManager.isUrlEligible(url) !== 'allowed') {
+            this.bufferPage(url, depth, 0, { securityError: 'blocked_by_domain_filter' });
+            return;
+        }
+        // Convert stored path to absolute URL for fetching.
+        // External/subdomain URLs are already absolute (UrlUtil.toPath returns them as-is).
+        const fetchUrl = UrlUtil.toAbsolute(url, this.rootOrigin);
+        try {
+            const prevNode = this.options.previousGraph?.nodes.get(url);
+            const res = await this.fetchPage(fetchUrl, depth, prevNode);
+            if (!res)
+                return;
+            const finalUrl = normalizeUrl(res.finalUrl, this.rootOrigin, this.options);
+            if (!finalUrl)
+                return;
+            const fullUrl = finalUrl; // Already absolute
+            const finalPath = this.toStorageUrl(finalUrl);
+            if (res.status === 304 && prevNode) {
+                this.handleCachedResponse(url, finalUrl, depth, prevNode);
+                return;
+            }
+            this.handleRedirects(res.redirectChain, depth);
+            const isStringStatus = typeof res.status === 'string';
+            if (isStringStatus || (typeof res.status === 'number' && res.status >= 300)) {
+                const statusNum = typeof res.status === 'number' ? res.status : 0;
+                this.bufferPage(finalPath, depth, statusNum, {
+                    security_error: isStringStatus ? res.status : undefined,
+                    retries: res.retries
+                });
+                this.bufferMetrics(finalPath, {
+                    crawl_status: isStringStatus ? res.status : 'fetched_error'
+                });
+                return;
+            }
+            if (res.status === 200) {
+                this.handleSuccessResponse(res, finalPath, fullUrl, depth, isBlocked);
+            }
+        }
+        catch (e) {
+            this.context.emit({ type: 'crawl:error', url, error: String(e), depth });
+        }
+    }
+    async run() {
+        // 1. Setup fetcher and basic modules
+        this.setupModules();
+        // 2. Initialize repositories, resolve URL (SSL/WWW), and set up site context
+        await this.initialize();
+        if (this.options.robots) {
+            this.robots = this.options.robots;
+        }
+        else {
+            await this.fetchRobots();
+        }
+        await this.seedQueue();
+        return new Promise((resolve) => {
+            const checkDone = async () => {
+                if (this.queue.length === 0 && this.active === 0 && this.pendingSitemaps === 0) {
+                    this.progressPhase = 'finalizing';
+                    this.emitProgress(true);
+                    await this.flushAll();
+                    this.snapshotRepo.updateSnapshotStatus(this.snapshotId, 'completed', {
+                        limit_reached: this.reachedLimit ? 1 : 0
+                    });
+                    this.snapshotRepo.pruneSnapshots(this.siteId, DEFAULTS.MAX_SNAPSHOTS, DEFAULTS.MAX_SINGLE_SNAPSHOTS, this.snapshotId);
+                    if (this.reusingSnapshot) {
+                        this.snapshotRepo.touchSnapshot(this.snapshotId);
+                    }
+                    resolve(this.snapshotId);
+                    return true;
+                }
+                return false;
+            };
+            const next = async () => {
+                if (await checkDone())
+                    return;
+                if (this.pagesCrawled >= this.options.limit) {
+                    this.reachedLimit = true;
+                    this.progressPhase = 'limit reached';
+                    this.emitProgress();
+                    if (this.active === 0) {
+                        this.context.emit({ type: 'crawl:limit-reached', limit: this.options.limit });
+                        this.progressPhase = 'finalizing';
+                        this.emitProgress(true);
+                        await this.flushAll();
+                        this.snapshotRepo.updateSnapshotStatus(this.snapshotId, 'completed', {
+                            limit_reached: 1
+                        });
+                        this.snapshotRepo.pruneSnapshots(this.siteId, DEFAULTS.MAX_SNAPSHOTS, DEFAULTS.MAX_SINGLE_SNAPSHOTS, this.snapshotId);
+                        if (this.reusingSnapshot) {
+                            this.snapshotRepo.touchSnapshot(this.snapshotId);
+                        }
+                        resolve(this.snapshotId);
+                    }
+                    return;
+                }
+                while (this.queue.length > 0 && this.active < this.concurrency && this.pagesCrawled < this.options.limit) {
+                    const item = this.queue.shift();
+                    if (this.visited.has(item.url))
+                        continue;
+                    // Robust robots check: reconstruct absolute URL since robots-parser needs full URLs,
+                    // not root-relative paths. Also check /path/ variant in case robots.txt uses trailing slash.
+                    const absUrlForRobots = UrlUtil.toAbsolute(item.url, this.rootOrigin);
+                    const isBlocked = this.robots && (!this.robots.isAllowed(absUrlForRobots, 'crawlith') ||
+                        (!absUrlForRobots.endsWith('/') && !this.robots.isAllowed(absUrlForRobots + '/', 'crawlith')));
+                    if (isBlocked) {
+                        if (this.options.debug) {
+                            console.log(`${chalk.yellow('⊘ Robots')} ${chalk.gray(item.url)}`);
+                        }
+                        // Tag as blocked for reporting
+                        this.bufferMetrics(item.url, {
+                            crawl_status: 'blocked_by_robots'
+                        });
+                        this.bufferPage(item.url, item.depth, 0);
+                        if (!this.options.ignoreRobots) {
+                            this.visited.add(item.url);
+                            this.pagesCrawled++;
+                            continue;
+                        }
+                    }
+                    this.active++;
+                    this.pagesCrawled++;
+                    this.visited.add(item.url);
+                    this.limitConcurrency(() => this.processPage(item, isBlocked)).finally(() => {
+                        this.active--;
+                        this.emitProgress();
+                        next();
+                    });
+                }
+                this.emitProgress();
+                await checkDone();
+            };
+            next();
+        });
+    }
+}

package/dist/crawler/extract.d.ts

@@ -1,5 +1,8 @@
 /**
  * Extracts all links from an HTML document.
  * Returns absolute URLs.
+ * @param html The HTML content string
+ * @param baseUrl The base URL to resolve relative links against
+ * @param onError Optional callback for handling extraction errors
  */
-export declare function extractLinks(html: string, baseUrl: string): string[];
+export declare function extractLinks(html: string, baseUrl: string, onError?: (error: unknown) => void): string[];