@crawlith/core 0.1.1 → 0.1.2

package/dist/crawler/crawler.js

@@ -1,11 +1,11 @@
-import chalk from 'chalk';
+import chalk from '../utils/chalk.js';
 import pLimit from 'p-limit';
 import robotsParser from 'robots-parser';
 import { Fetcher } from './fetcher.js';
 import { Parser } from './parser.js';
 import { Sitemap } from './sitemap.js';
-import { normalizeUrl } from './normalize.js';
-import { TrapDetector } from './trap.js';
+import { normalizeUrl, UrlUtil } from './normalize.js';
+import { UrlResolver } from './resolver.js';
 import { ScopeManager } from '../core/scope/scopeManager.js';
 import { getDb } from '../db/index.js';
 import { SiteRepository } from '../db/repositories/SiteRepository.js';
@@ -15,6 +15,7 @@ import { EdgeRepository } from '../db/repositories/EdgeRepository.js';
 import { MetricsRepository } from '../db/repositories/MetricsRepository.js';
 import { analyzeContent, calculateThinContentScore } from '../analysis/content.js';
 import { analyzeLinks } from '../analysis/links.js';
+import { DEFAULTS } from '../constants.js';
 // Fallback context for backward compatibility or when no context is provided
 const nullContext = {
     emit: (event) => {
@@ -32,6 +33,7 @@ export class Crawler {
     startUrl;
     options;
     context;
+    registry;
     visited;
     uniqueQueue;
     queue;
@@ -50,6 +52,8 @@ export class Crawler {
     // Site/Snapshot info
     siteId = null;
     snapshotId = null;
+    reusingSnapshot = false;
+    runType = 'completed';
     rootOrigin = '';
     // Discovery tracking
     discoveryDepths = new Map();
@@ -57,27 +61,34 @@ export class Crawler {
     pageBuffer = new Map();
     edgeBuffer = [];
     metricsBuffer = [];
+    pendingSitemaps = 0;
+    edgesFound = 0;
+    lastProgressEmitAt = 0;
+    progressPhase = 'crawling';
     // Modules
     scopeManager = null;
     fetcher = null;
     parser = null;
     sitemapFetcher = null;
-    trapDetector = null;
     robots = null;
     constructor(startUrl, options, context) {
         this.startUrl = startUrl;
         this.options = options;
         this.context = context || nullContext;
+        this.registry = options.registry;
         this.visited = new Set();
         this.uniqueQueue = new Set();
         this.queue = [];
         this.active = 0;
         this.pagesCrawled = 0;
         this.reachedLimit = false;
-        this.maxDepthInCrawl = Math.min(options.depth, 10);
-        this.concurrency = Math.min(options.concurrency || 2, 10);
+        this.maxDepthInCrawl = Math.min(options.depth || DEFAULTS.MAX_DEPTH, DEFAULTS.MAX_DEPTH_LIMIT);
+        this.concurrency = Math.min(options.concurrency || DEFAULTS.CONCURRENCY, DEFAULTS.CONCURRENCY_LIMIT);
         this.limitConcurrency = pLimit(this.concurrency);
     }
+    toStorageUrl(url) {
+        return UrlUtil.isInternal(url, this.rootOrigin) ? UrlUtil.toPath(url, this.rootOrigin) : url;
+    }
     async initialize() {
         const db = getDb();
         this.siteRepo = new SiteRepository(db);
@@ -85,41 +96,64 @@ export class Crawler {
         this.pageRepo = new PageRepository(db);
         this.edgeRepo = new EdgeRepository(db);
         this.metricsRepo = new MetricsRepository(db);
-        const rootUrl = normalizeUrl(this.startUrl, '', { stripQuery: this.options.stripQuery });
+        // Use resolver to find canonical origin and SSL
+        const resolver = new UrlResolver();
+        const tempFetcher = new Fetcher({ userAgent: this.options.userAgent, rate: this.options.rate });
+        const resolved = await resolver.resolve(this.startUrl, tempFetcher);
+        this.rootOrigin = resolved.url;
+        // Use the resolved absolute URL as the base — NOT this.startUrl which may be
+        // a bare domain (e.g. 'callforpaper.org') that would be treated as a relative
+        // path when passed to normalizeUrl, producing '/callforpaper.org'.
+        const rootUrl = normalizeUrl(this.rootOrigin, '', { stripQuery: this.options.stripQuery });
         if (!rootUrl)
             throw new Error('Invalid start URL');
-        const urlObj = new URL(rootUrl);
+        const urlObj = new URL(this.rootOrigin);
         const domain = urlObj.hostname.replace('www.', '');
         const site = this.siteRepo.firstOrCreateSite(domain);
         this.siteId = site.id;
-        const type = this.options.snapshotType || (this.options.previousGraph ? 'incremental' : 'full');
-        this.snapshotId = this.snapshotRepo.createSnapshot(this.siteId, type);
+        // Persist the resolved preferred URL and SSL status
+        this.siteRepo.updateSitePreference(this.siteId, {
+            preferred_url: this.rootOrigin,
+            ssl: this.rootOrigin.startsWith('https') ? 1 : 0
+        });
         this.rootOrigin = urlObj.origin;
-        this.startUrl = rootUrl;
-        // Seed discovery depth for root
-        this.discoveryDepths.set(this.startUrl, 0);
-    }
-    setupModules() {
+        // Keep storage path-first for internal URLs and reconcile any legacy absolute rows.
+        this.pageRepo.reconcileInternalUrls(this.siteId, this.rootOrigin);
+        this.startUrl = this.toStorageUrl(rootUrl);
+        // Now that rootOrigin is resolved, initialize ScopeManager with the correct absolute origin
         this.scopeManager = new ScopeManager({
             allowedDomains: this.options.allowedDomains || [],
             deniedDomains: this.options.deniedDomains || [],
             includeSubdomains: this.options.includeSubdomains || false,
-            rootUrl: this.startUrl
+            rootUrl: this.rootOrigin
         });
+        // Update fetcher with the now-initialized scopeManager
+        if (this.fetcher) {
+            this.fetcher.scopeManager = this.scopeManager;
+        }
+        // Every scan now creates a new snapshot (no reuse)
+        const runType = this.options.snapshotRunType || (this.options.previousGraph ? 'incremental' : 'completed');
+        this.snapshotId = this.snapshotRepo.createSnapshot(this.siteId, runType);
+        this.runType = runType;
+        // Expose snapshot context for plugins that persist per-snapshot data.
+        this.context.snapshotId = this.snapshotId;
+        // Seed discovery depth for root
+        this.discoveryDepths.set(this.startUrl, 0);
+    }
+    setupModules() {
         this.fetcher = new Fetcher({
             rate: this.options.rate,
             proxyUrl: this.options.proxyUrl,
-            scopeManager: this.scopeManager,
+            scopeManager: this.scopeManager ?? undefined,
             maxRedirects: this.options.maxRedirects,
             userAgent: this.options.userAgent
         });
         this.parser = new Parser();
-        this.sitemapFetcher = new Sitemap(this.context);
-        this.trapDetector = new TrapDetector();
+        this.sitemapFetcher = new Sitemap(this.context, this.fetcher);
     }
     async fetchRobots() {
+        const robotsUrl = new URL('/robots.txt', this.rootOrigin).toString();
         try {
-            const robotsUrl = new URL('/robots.txt', this.rootOrigin).toString();
             const res = await this.fetcher.fetch(robotsUrl, { maxBytes: 500000 });
             if (res && typeof res.status === 'number' && res.status >= 200 && res.status < 300) {
                 this.robots = robotsParser(robotsUrl, res.body);
@@ -139,20 +173,22 @@ export class Crawler {
             return false;
         if (this.scopeManager.isUrlEligible(url) !== 'allowed')
             return false;
-        if (this.options.detectTraps) {
-            const trap = this.trapDetector.checkTrap(url, depth);
-            if (trap.risk > 0.8)
+        if (this.registry) {
+            const allowed = this.registry.runSyncBailHook('shouldEnqueueUrl', this.context, url, depth);
+            if (allowed === false)
                 return false;
         }
         return true;
     }
-    addToQueue(u, d) {
+    addToQueue(u, d, data = {}) {
         if (this.scopeManager.isUrlEligible(u) !== 'allowed')
             return;
         if (!this.uniqueQueue.has(u)) {
             this.uniqueQueue.add(u);
             this.queue.push({ url: u, depth: d });
             this.context.emit({ type: 'queue:enqueue', url: u, depth: d });
+            this.emitProgress();
+            this.bufferPage(u, d, 0, data);
             const currentDiscovery = this.discoveryDepths.get(u);
             if (currentDiscovery === undefined || d < currentDiscovery) {
                 this.discoveryDepths.set(u, d);
@@ -160,26 +196,72 @@ export class Crawler {
         }
     }
     async seedQueue() {
-        // Seed from Sitemap
-        if (this.options.sitemap) {
-            try {
-                const sitemapUrl = this.options.sitemap === 'true' ? new URL('/sitemap.xml', this.rootOrigin).toString() : this.options.sitemap;
-                if (sitemapUrl.startsWith('http')) {
-                    this.context.emit({ type: 'info', message: 'Fetching sitemap', context: { url: sitemapUrl } });
-                    const sitemapUrls = await this.sitemapFetcher.fetch(sitemapUrl);
-                    for (const u of sitemapUrls) {
-                        const normalized = normalizeUrl(u, '', this.options);
-                        if (normalized)
-                            this.addToQueue(normalized, 0);
-                    }
-                }
+        // Seed from startUrl first to ensure it's prioritized in the queue
+        this.addToQueue(this.startUrl, 0);
+        const sitemapsToFetch = new Set();
+        // 1. Explicitly configured sitemap
+        if (this.options.sitemap && this.runType !== 'single') {
+            const explicitUrl = this.options.sitemap === 'true' || this.options.sitemap === true
+                ? new URL('/sitemap.xml', this.rootOrigin).toString()
+                : this.options.sitemap;
+            if (typeof explicitUrl === 'string' && explicitUrl.startsWith('http')) {
+                sitemapsToFetch.add(explicitUrl);
             }
-            catch (e) {
-                this.context.emit({ type: 'warn', message: 'Sitemap fetch failed', context: e });
+        }
+        // 2. Discover sitemaps from robots.txt (unless explicitly disabled)
+        // Only auto-fetch on the FIRST real crawl (full/incremental).
+        // page --live reuses snapshots and should NOT trigger sitemap fetch.
+        const isFirstFullCrawl = this.runType !== 'single' && !this.snapshotRepo?.hasFullCrawl(this.siteId);
+        if (this.options.sitemap !== false && (this.options.sitemap || isFirstFullCrawl) && this.robots && this.runType !== 'single') {
+            const robotsSitemaps = this.robots.getSitemaps();
+            for (const s of robotsSitemaps) {
+                if (s)
+                    sitemapsToFetch.add(s);
+            }
+        }
+        // Process all discovered sitemaps in background
+        if (sitemapsToFetch.size > 0) {
+            for (const sitemapUrl of sitemapsToFetch) {
+                this.pendingSitemaps++;
+                // KICK OFF BACKGROUND TASK (Un-awaited)
+                (async () => {
+                    try {
+                        this.context.emit({ type: 'debug', message: 'Fetching sitemap in background', context: { url: sitemapUrl } });
+                        const sitemapUrls = await this.sitemapFetcher.fetch(sitemapUrl);
+                        if (sitemapUrls.length > 0) {
+                            this.context.emit({ type: 'debug', message: `Mapping ${sitemapUrls.length} URLs from sitemap... (Background)` });
+                            const sitemapEntries = sitemapUrls.map(u => {
+                                const normalized = normalizeUrl(u, this.rootOrigin, this.options);
+                                if (!normalized)
+                                    return null;
+                                const path = this.toStorageUrl(normalized);
+                                return {
+                                    site_id: this.siteId,
+                                    normalized_url: path,
+                                    first_seen_snapshot_id: this.snapshotId,
+                                    last_seen_snapshot_id: this.snapshotId,
+                                    discovered_via_sitemap: 1,
+                                    depth: 0,
+                                    http_status: 0
+                                };
+                            }).filter((p) => p !== null);
+                            // Bulk register to DB
+                            this.pageRepo.upsertMany(sitemapEntries);
+                            // Add to queue for Actual Crawling
+                            for (const entry of sitemapEntries) {
+                                this.addToQueue(entry.normalized_url, 0, { discovered_via_sitemap: 1 });
+                            }
+                        }
+                    }
+                    catch (e) {
+                        this.context.emit({ type: 'warn', message: 'Sitemap fetch failed', context: { url: sitemapUrl, error: String(e) } });
+                    }
+                    finally {
+                        this.pendingSitemaps--;
+                    }
+                })();
             }
         }
-        // Seed from startUrl
-        this.addToQueue(this.startUrl, 0);
     }
     bufferPage(url, depth, status, data = {}) {
         const existing = this.pageBuffer.get(url);
@@ -226,10 +308,27 @@ export class Crawler {
     }
     bufferEdge(sourceUrl, targetUrl, weight = 1.0, rel = 'internal') {
         this.edgeBuffer.push({ sourceUrl, targetUrl, weight, rel });
+        this.edgesFound += 1;
+        this.emitProgress();
         if (this.edgeBuffer.length >= 100) {
             this.flushEdges();
         }
     }
+    emitProgress(force = false) {
+        const now = Date.now();
+        if (!force && now - this.lastProgressEmitAt < 200)
+            return;
+        this.lastProgressEmitAt = now;
+        this.context.emit({
+            type: 'crawl:progress',
+            pagesCrawled: this.pagesCrawled,
+            queued: this.queue.length,
+            active: this.active,
+            nodesFound: this.uniqueQueue.size,
+            edgesFound: this.edgesFound,
+            phase: this.progressPhase
+        });
+    }
     flushEdges() {
         if (this.edgeBuffer.length === 0)
             return;
@@ -237,6 +336,13 @@ export class Crawler {
         this.flushPages();
         const identities = this.pageRepo.getPagesIdentityBySnapshot(this.snapshotId);
         const urlToId = new Map(identities.map(p => [p.normalized_url, p.id]));
+        // When reusing a snapshot, clean up stale edges for pages being re-crawled
+        if (this.reusingSnapshot) {
+            const sourcePageIds = new Set(this.edgeBuffer.map(e => urlToId.get(e.sourceUrl)).filter((id) => id !== undefined));
+            for (const pageId of sourcePageIds) {
+                this.edgeRepo.deleteEdgesForPage(this.snapshotId, pageId);
+            }
+        }
         const edgesToInsert = this.edgeBuffer
             .map(e => ({
             snapshot_id: this.snapshotId,
@@ -270,18 +376,23 @@ export class Crawler {
             return {
                 snapshot_id: this.snapshotId,
                 page_id: pageId,
-                authority_score: null,
-                hub_score: null,
-                pagerank: null,
-                pagerank_score: null,
-                link_role: null,
                 crawl_status: null,
                 word_count: null,
                 thin_content_score: null,
                 external_link_ratio: null,
-                orphan_score: null,
+                pagerank_score: null,
+                hub_score: null,
+                auth_score: null,
+                link_role: null,
                 duplicate_cluster_id: null,
                 duplicate_type: null,
+                cluster_id: null,
+                soft404_score: null,
+                heading_score: null,
+                orphan_score: null,
+                orphan_type: null,
+                impact_level: null,
+                heading_data: null,
                 is_cluster_primary: 0,
                 ...item.data
             };
@@ -322,31 +433,39 @@ export class Crawler {
         }
     }
     handleCachedResponse(url, finalUrl, depth, prevNode) {
-        this.bufferPage(finalUrl, depth, 200, {
+        const path = url;
+        const finalPath = this.toStorageUrl(finalUrl);
+        this.bufferPage(finalPath, depth, prevNode.status, {
             html: prevNode.html,
             canonical_url: prevNode.canonical,
+            noindex: prevNode.noindex ? 1 : 0,
+            nofollow: prevNode.nofollow ? 1 : 0,
             content_hash: prevNode.contentHash,
             simhash: prevNode.simhash,
             etag: prevNode.etag,
-            last_modified: prevNode.lastModified,
-            noindex: prevNode.noindex ? 1 : 0,
-            nofollow: prevNode.nofollow ? 1 : 0
+            last_modified: prevNode.lastModified
         });
-        this.bufferMetrics(finalUrl, {
-            crawl_status: 'cached'
+        this.bufferMetrics(finalPath, {
+            crawl_status: 'cached',
+            word_count: prevNode.wordCount,
+            thin_content_score: prevNode.thinContentScore,
+            external_link_ratio: prevNode.externalLinkRatio
         });
         // Re-discovery links from previous graph to continue crawling if needed
         const prevLinks = this.options.previousGraph?.getEdges()
-            .filter(e => e.source === url)
+            .filter(e => e.source === path)
             .map(e => e.target);
         if (prevLinks) {
             for (const link of prevLinks) {
-                const normalizedLink = normalizeUrl(link, '', this.options);
-                if (normalizedLink && normalizedLink !== finalUrl) {
-                    this.bufferPage(normalizedLink, depth + 1, 0);
-                    this.bufferEdge(finalUrl, normalizedLink, 1.0, 'internal');
-                    if (this.shouldEnqueue(normalizedLink, depth + 1)) {
-                        this.addToQueue(normalizedLink, depth + 1);
+                const normalizedLink = normalizeUrl(link, this.rootOrigin, this.options);
+                if (normalizedLink) {
+                    const path = this.toStorageUrl(normalizedLink);
+                    if (path !== url) {
+                        this.bufferPage(path, depth + 1, 0);
+                        this.bufferEdge(url, path, 1.0, 'internal');
+                        if (this.shouldEnqueue(path, depth + 1)) {
+                            this.addToQueue(path, depth + 1);
+                        }
                     }
                 }
             }
@@ -354,40 +473,53 @@ export class Crawler {
     }
     handleRedirects(chain, depth) {
         for (const step of chain) {
-            const source = normalizeUrl(step.url, '', this.options);
-            const target = normalizeUrl(step.target, '', this.options);
-            if (source && target) {
-                this.bufferPage(source, depth, step.status);
-                this.bufferPage(target, depth, 0);
-                this.bufferEdge(source, target);
+            const sourceAbs = normalizeUrl(step.url, this.rootOrigin, this.options);
+            const targetAbs = normalizeUrl(step.target, this.rootOrigin, this.options);
+            if (sourceAbs && targetAbs) {
+                const sourcePath = this.toStorageUrl(sourceAbs);
+                const targetPath = this.toStorageUrl(targetAbs);
+                const sourceInternal = UrlUtil.isInternal(sourceAbs, this.rootOrigin);
+                const targetInternal = UrlUtil.isInternal(targetAbs, this.rootOrigin);
+                this.bufferPage(sourcePath, depth, step.status, { is_internal: sourceInternal ? 1 : 0 });
+                this.bufferPage(targetPath, depth, 0, { is_internal: targetInternal ? 1 : 0 });
+                this.bufferEdge(sourcePath, targetPath, 1.0, targetInternal ? 'internal' : 'external');
             }
         }
     }
-    handleSuccessResponse(res, finalUrl, depth, isBlocked = false) {
+    handleSuccessResponse(res, path, absoluteUrl, depth, isBlocked = false) {
         const contentTypeHeader = res.headers['content-type'];
         const contentType = Array.isArray(contentTypeHeader) ? contentTypeHeader[0] : (contentTypeHeader || '');
         if (!contentType || !contentType.toLowerCase().includes('text/html')) {
-            this.bufferPage(finalUrl, depth, typeof res.status === 'number' ? res.status : 0);
+            this.bufferPage(path, depth, typeof res.status === 'number' ? res.status : 0);
             return;
         }
-        const parseResult = this.parser.parse(res.body, finalUrl, res.status);
-        this.bufferPage(finalUrl, depth, res.status, {
+        const parseResult = this.parser.parse(res.body, absoluteUrl, res.status);
+        if (this.registry) {
+            this.registry.runHook('onPageParsed', this.context, {
+                url: absoluteUrl,
+                status: res.status,
+                depth: depth,
+                headers: res.headers,
+                ...parseResult
+            });
+        }
+        this.bufferPage(path, depth, res.status, {
             html: parseResult.html,
             canonical_url: parseResult.canonical || undefined,
             noindex: parseResult.noindex ? 1 : 0,
             nofollow: parseResult.nofollow ? 1 : 0,
             content_hash: parseResult.contentHash,
             simhash: parseResult.simhash,
-            soft404_score: parseResult.soft404Score,
             etag: res.etag,
             last_modified: res.lastModified,
-            retries: res.retries
+            retries: res.retries,
+            bytes_received: res.bytesReceived
         });
         try {
             const contentAnalysis = analyzeContent(parseResult.html);
-            const linkAnalysis = analyzeLinks(parseResult.html, finalUrl, this.rootOrigin);
+            const linkAnalysis = analyzeLinks(parseResult.html, absoluteUrl, this.rootOrigin);
             const thinScore = calculateThinContentScore(contentAnalysis, 0);
-            this.bufferMetrics(finalUrl, {
+            this.bufferMetrics(path, {
                 crawl_status: isBlocked ? 'blocked_by_robots' : 'fetched',
                 word_count: contentAnalysis.wordCount,
                 thin_content_score: thinScore,
@@ -395,15 +527,19 @@ export class Crawler {
             });
         }
         catch (e) {
-            this.context.emit({ type: 'error', message: 'Error calculating per-page metrics', error: e, context: { url: finalUrl } });
+            this.context.emit({ type: 'error', message: 'Error calculating per-page metrics', error: e, context: { url: absoluteUrl } });
         }
         for (const linkItem of parseResult.links) {
-            const normalizedLink = normalizeUrl(linkItem.url, '', this.options);
-            if (normalizedLink && normalizedLink !== finalUrl) {
-                this.bufferPage(normalizedLink, depth + 1, 0);
-                this.bufferEdge(finalUrl, normalizedLink, 1.0, 'internal');
-                if (this.shouldEnqueue(normalizedLink, depth + 1)) {
-                    this.addToQueue(normalizedLink, depth + 1);
+            const normalizedLink = normalizeUrl(linkItem.url, absoluteUrl, this.options);
+            if (normalizedLink) {
+                const targetPath = this.toStorageUrl(normalizedLink);
+                if (targetPath !== path) {
+                    const isInternal = UrlUtil.isInternal(normalizedLink, this.rootOrigin);
+                    this.bufferPage(targetPath, depth + 1, 0, { is_internal: isInternal ? 1 : 0 });
+                    this.bufferEdge(path, targetPath, 1.0, isInternal ? 'internal' : 'external');
+                    if (isInternal && this.shouldEnqueue(targetPath, depth + 1)) {
+                        this.addToQueue(targetPath, depth + 1);
+                    }
                 }
             }
         }
@@ -414,14 +550,19 @@ export class Crawler {
             this.bufferPage(url, depth, 0, { securityError: 'blocked_by_domain_filter' });
             return;
         }
+        // Convert stored path to absolute URL for fetching.
+        // External/subdomain URLs are already absolute (UrlUtil.toPath returns them as-is).
+        const fetchUrl = UrlUtil.toAbsolute(url, this.rootOrigin);
         try {
             const prevNode = this.options.previousGraph?.nodes.get(url);
-            const res = await this.fetchPage(url, depth, prevNode);
+            const res = await this.fetchPage(fetchUrl, depth, prevNode);
             if (!res)
                 return;
-            const finalUrl = normalizeUrl(res.finalUrl, '', this.options);
+            const finalUrl = normalizeUrl(res.finalUrl, this.rootOrigin, this.options);
             if (!finalUrl)
                 return;
+            const fullUrl = finalUrl; // Already absolute
+            const finalPath = this.toStorageUrl(finalUrl);
             if (res.status === 304 && prevNode) {
                 this.handleCachedResponse(url, finalUrl, depth, prevNode);
                 return;
@@ -430,17 +571,17 @@ export class Crawler {
             const isStringStatus = typeof res.status === 'string';
             if (isStringStatus || (typeof res.status === 'number' && res.status >= 300)) {
                 const statusNum = typeof res.status === 'number' ? res.status : 0;
-                this.bufferPage(finalUrl, depth, statusNum, {
+                this.bufferPage(finalPath, depth, statusNum, {
                     security_error: isStringStatus ? res.status : undefined,
                     retries: res.retries
                 });
-                this.bufferMetrics(finalUrl, {
+                this.bufferMetrics(finalPath, {
                     crawl_status: isStringStatus ? res.status : 'fetched_error'
                 });
                 return;
             }
             if (res.status === 200) {
-                this.handleSuccessResponse(res, finalUrl, depth, isBlocked);
+                this.handleSuccessResponse(res, finalPath, fullUrl, depth, isBlocked);
             }
         }
         catch (e) {
@@ -448,17 +589,30 @@ export class Crawler {
         }
     }
     async run() {
-        await this.initialize();
+        // 1. Setup fetcher and basic modules
         this.setupModules();
-        await this.fetchRobots();
+        // 2. Initialize repositories, resolve URL (SSL/WWW), and set up site context
+        await this.initialize();
+        if (this.options.robots) {
+            this.robots = this.options.robots;
+        }
+        else {
+            await this.fetchRobots();
+        }
         await this.seedQueue();
         return new Promise((resolve) => {
             const checkDone = async () => {
-                if (this.queue.length === 0 && this.active === 0) {
+                if (this.queue.length === 0 && this.active === 0 && this.pendingSitemaps === 0) {
+                    this.progressPhase = 'finalizing';
+                    this.emitProgress(true);
                     await this.flushAll();
                     this.snapshotRepo.updateSnapshotStatus(this.snapshotId, 'completed', {
                         limit_reached: this.reachedLimit ? 1 : 0
                     });
+                    this.snapshotRepo.pruneSnapshots(this.siteId, DEFAULTS.MAX_SNAPSHOTS, DEFAULTS.MAX_SINGLE_SNAPSHOTS, this.snapshotId);
+                    if (this.reusingSnapshot) {
+                        this.snapshotRepo.touchSnapshot(this.snapshotId);
+                    }
                     resolve(this.snapshotId);
                     return true;
                 }
@@ -469,12 +623,20 @@ export class Crawler {
                     return;
                 if (this.pagesCrawled >= this.options.limit) {
                     this.reachedLimit = true;
+                    this.progressPhase = 'limit reached';
+                    this.emitProgress();
                     if (this.active === 0) {
+                        this.context.emit({ type: 'crawl:limit-reached', limit: this.options.limit });
+                        this.progressPhase = 'finalizing';
+                        this.emitProgress(true);
                         await this.flushAll();
                         this.snapshotRepo.updateSnapshotStatus(this.snapshotId, 'completed', {
                             limit_reached: 1
                         });
-                        this.context.emit({ type: 'crawl:limit-reached', limit: this.options.limit });
+                        this.snapshotRepo.pruneSnapshots(this.siteId, DEFAULTS.MAX_SNAPSHOTS, DEFAULTS.MAX_SINGLE_SNAPSHOTS, this.snapshotId);
+                        if (this.reusingSnapshot) {
+                            this.snapshotRepo.touchSnapshot(this.snapshotId);
+                        }
                         resolve(this.snapshotId);
                     }
                     return;
@@ -483,10 +645,11 @@ export class Crawler {
                     const item = this.queue.shift();
                     if (this.visited.has(item.url))
                         continue;
-                    // Robust robots check: if path doesn't end in /, check both /path and /path/
-                    // to handle cases where normalization stripped a slash that robots.txt relies on.
-                    const isBlocked = this.robots && (!this.robots.isAllowed(item.url, 'crawlith') ||
-                        (!item.url.endsWith('/') && !this.robots.isAllowed(item.url + '/', 'crawlith')));
+                    // Robust robots check: reconstruct absolute URL since robots-parser needs full URLs,
+                    // not root-relative paths. Also check /path/ variant in case robots.txt uses trailing slash.
+                    const absUrlForRobots = UrlUtil.toAbsolute(item.url, this.rootOrigin);
+                    const isBlocked = this.robots && (!this.robots.isAllowed(absUrlForRobots, 'crawlith') ||
+                        (!absUrlForRobots.endsWith('/') && !this.robots.isAllowed(absUrlForRobots + '/', 'crawlith')));
                     if (isBlocked) {
                         if (this.options.debug) {
                             console.log(`${chalk.yellow('⊘ Robots')} ${chalk.gray(item.url)}`);
@@ -507,9 +670,11 @@ export class Crawler {
                     this.visited.add(item.url);
                     this.limitConcurrency(() => this.processPage(item, isBlocked)).finally(() => {
                         this.active--;
+                        this.emitProgress();
                         next();
                     });
                 }
+                this.emitProgress();
                 await checkDone();
             };
             next();

package/dist/crawler/fetcher.d.ts

@@ -23,7 +23,7 @@ export interface FetchOptions {
     crawlDelay?: number;
 }
 export declare class Fetcher {
-    private userAgent;
+    userAgent: string;
     private rateLimiter;
     private proxyAdapter;
     private secureDispatcher;

package/dist/crawler/fetcher.js

@@ -6,16 +6,16 @@ import { RetryPolicy } from '../core/network/retryPolicy.js';
 import { ResponseLimiter } from '../core/network/responseLimiter.js';
 import { RedirectController } from '../core/network/redirectController.js';
 import { ProxyAdapter } from '../core/network/proxyAdapter.js';
-import { version } from '../utils/version.js';
+import { DEFAULTS } from '../constants.js';
 export class Fetcher {
-    userAgent = 'crawlith/1.0';
+    userAgent = DEFAULTS.USER_AGENT;
     rateLimiter;
     proxyAdapter;
     secureDispatcher;
     scopeManager;
     maxRedirects;
     constructor(options = {}) {
-        this.rateLimiter = new RateLimiter(options.rate || 2);
+        this.rateLimiter = new RateLimiter(options.rate || DEFAULTS.RATE_LIMIT);
         this.proxyAdapter = new ProxyAdapter(options.proxyUrl);
         if (this.proxyAdapter.dispatcher) {
             this.secureDispatcher = this.proxyAdapter.dispatcher;
@@ -24,11 +24,11 @@ export class Fetcher {
             this.secureDispatcher = IPGuard.getSecureDispatcher();
         }
         this.scopeManager = options.scopeManager;
-        this.maxRedirects = Math.min(options.maxRedirects ?? 2, 11);
-        this.userAgent = options.userAgent || `crawlith/${version}`;
+        this.maxRedirects = Math.min(options.maxRedirects ?? DEFAULTS.MAX_REDIRECTS, DEFAULTS.MAX_REDIRECTS_LIMIT);
+        this.userAgent = options.userAgent || DEFAULTS.USER_AGENT;
     }
     async fetch(url, options = {}) {
-        const maxBytes = options.maxBytes || 2000000;
+        const maxBytes = options.maxBytes || DEFAULTS.MAX_BYTES;
         const redirectChain = [];
         const redirectController = new RedirectController(this.maxRedirects, url);
         let currentUrl = url;