@houtini/seo-crawler-mcp 2.0.1

package/build/core/CrawlOrchestrator.js

@@ -0,0 +1,279 @@
+import { HttpCrawler, RequestQueue, Configuration, log } from 'crawlee';
+import { MemoryStorage } from '@crawlee/memory-storage';
+import { load } from 'cheerio';
+import { debug } from '../utils/debug.js';
+export class CrawlOrchestrator {
+    config;
+    urlManager;
+    contentExtractor;
+    linkExtractor;
+    storage;
+    crawler;
+    metadata;
+    linkBuffer = [];
+    LINK_BUFFER_SIZE = 100;
+    memoryStorage;
+    constructor(config, urlManager, contentExtractor, linkExtractor, storage) {
+        this.config = config;
+        this.urlManager = urlManager;
+        this.contentExtractor = contentExtractor;
+        this.linkExtractor = linkExtractor;
+        this.storage = storage;
+        log.setLevel(log.LEVELS.OFF);
+        this.metadata = this.createInitialMetadata();
+    }
+    createInitialMetadata() {
+        return {
+            crawlId: this.config.crawlId,
+            status: 'queued',
+            startedAt: null,
+            completedAt: null,
+            duration: null,
+            stats: {
+                discovered: 0,
+                crawled: 0,
+                failed: 0,
+                skipped: 0,
+                depth: 0,
+                speed: 0
+            },
+            errors: []
+        };
+    }
+    async initializeCrawler() {
+        const storageDir = `./crawlee-storage-${this.config.crawlId}`;
+        this.memoryStorage = new MemoryStorage({ localDataDirectory: storageDir });
+        const configuration = new Configuration({
+            storageClient: this.memoryStorage,
+            persistStorage: false,
+        });
+        const requestQueue = await RequestQueue.open(undefined, { config: configuration });
+        const userAgents = {
+            chrome: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+            googlebot: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+        };
+        const userAgent = userAgents[this.config.userAgent] || userAgents.chrome;
+        const crawlerConfig = {
+            maxRequestsPerCrawl: this.config.maxPages,
+            maxConcurrency: 20,
+            minConcurrency: 5,
+            maxRequestRetries: 5,
+            requestHandlerTimeoutSecs: this.config.timeout / 1000,
+            navigationTimeoutSecs: 30,
+            additionalMimeTypes: ['text/html', 'application/xhtml+xml'],
+            requestQueue,
+            preNavigationHooks: [
+                async ({ request }) => {
+                    request.headers = {
+                        'User-Agent': userAgent,
+                        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                        'Accept-Language': 'en-US,en;q=0.9',
+                        'Accept-Encoding': 'gzip, deflate, br',
+                        'Connection': 'keep-alive',
+                        'Upgrade-Insecure-Requests': '1'
+                    };
+                }
+            ],
+            requestHandler: async (context) => {
+                const { request, response, body, crawler } = context;
+                if (!response || !body)
+                    return;
+                const $ = load(body.toString());
+                await this.processPage(request.url, body.toString(), $, response, crawler);
+            },
+            failedRequestHandler: async (context, error) => {
+                const { request } = context;
+                await this.handleFailedRequest(request, error);
+            }
+        };
+        this.crawler = new HttpCrawler(crawlerConfig);
+    }
+    async run() {
+        try {
+            debug('[ORCH] Starting orchestrator.run()');
+            debug('[ORCH] Config:', JSON.stringify(this.config, null, 2));
+            await this.initializeCrawler();
+            debug('[ORCH] Crawler initialized with isolated MemoryStorage');
+            await this.storage.initialize();
+            debug('[ORCH] Storage initialized');
+            this.metadata.status = 'running';
+            this.metadata.startedAt = new Date().toISOString();
+            await this.storage.saveMetadata(this.metadata, this.config);
+            debug('[ORCH] Metadata saved, status=running');
+            this.urlManager.addDiscovered(this.config.startUrl, 0);
+            debug('[ORCH] Start URL added to UrlManager');
+            debug('[ORCH] About to call crawler.run()...');
+            await this.crawler.run([this.config.startUrl]);
+            debug('[ORCH] Crawler.run() completed');
+            if (this.linkBuffer.length > 0) {
+                await this.storage.saveLinkData(this.linkBuffer);
+                this.linkBuffer = [];
+            }
+            const endTime = Date.now();
+            const startTime = new Date(this.metadata.startedAt).getTime();
+            this.metadata.status = 'completed';
+            this.metadata.completedAt = new Date().toISOString();
+            this.metadata.duration = endTime - startTime;
+            this.metadata.stats.depth = this.urlManager.getMaxDepth();
+            this.metadata.stats.speed = this.metadata.duration > 0
+                ? (this.metadata.stats.crawled / (this.metadata.duration / 1000))
+                : 0;
+        }
+        catch (error) {
+            console.error('[ORCH ERROR] Fatal error in run():', error);
+            console.error('[ORCH ERROR] Stack:', error.stack);
+            this.metadata.status = 'failed';
+            this.metadata.errors.push({
+                url: '',
+                errorType: 'unknown',
+                message: error.message || 'Unknown error',
+                timestamp: new Date().toISOString()
+            });
+        }
+        finally {
+            if (this.memoryStorage) {
+                try {
+                    await this.memoryStorage.purge();
+                    debug('[ORCH] MemoryStorage cleaned up successfully');
+                }
+                catch (cleanupError) {
+                    console.error('[ORCH WARN] Storage cleanup failed:', cleanupError.message);
+                }
+            }
+        }
+        await this.storage.saveMetadata(this.metadata, this.config);
+        return this.metadata;
+    }
+    async processPage(url, html, $, response, crawler) {
+        this.urlManager.markVisited(url);
+        const pageData = this.contentExtractor.extract(url, html, $, {
+            crawlId: this.config.crawlId,
+            depth: this.urlManager.getDepth(url),
+            statusCode: response.status || 200,
+            contentType: response.headers?.['content-type'] || 'text/html',
+            responseTime: 0,
+            size: html.length,
+            isInternal: this.urlManager.isInternal(url),
+            linkedFrom: this.urlManager.getSourcePages(url),
+            redirects: []
+        }, response);
+        const links = this.linkExtractor.extract($, url, this.config.crawlId);
+        const currentDepth = this.urlManager.getDepth(url);
+        const linksToAdd = [];
+        for (const link of links) {
+            if (this.shouldCrawlUrl(link.targetUrl)) {
+                if (currentDepth < this.config.maxDepth) {
+                    if (!this.urlManager.isDiscovered(link.targetUrl)) {
+                        this.urlManager.addDiscovered(link.targetUrl, currentDepth + 1, url);
+                    }
+                    linksToAdd.push(link.targetUrl);
+                }
+                else {
+                    this.metadata.stats.skipped++;
+                }
+            }
+            else {
+                this.metadata.stats.skipped++;
+            }
+        }
+        if (linksToAdd.length > 0) {
+            await crawler.addRequests(linksToAdd);
+        }
+        await this.storage.savePageData(pageData);
+        this.linkBuffer.push(...links);
+        if (this.linkBuffer.length >= this.LINK_BUFFER_SIZE) {
+            await this.storage.saveLinkData(this.linkBuffer);
+            this.linkBuffer = [];
+        }
+        this.metadata.stats.crawled++;
+        this.metadata.stats.discovered = this.urlManager.getTotalDiscovered();
+        if (this.metadata.stats.crawled % 10 === 0) {
+            await this.storage.updateMetadata(this.metadata);
+        }
+    }
+    async handleFailedRequest(request, error) {
+        this.metadata.stats.failed++;
+        const errorType = this.categorizeError(error);
+        const errorMessage = error.message || 'Unknown error';
+        const errorMessages = request.errorMessages || [];
+        const fullErrorDetails = errorMessages.length > 0
+            ? `${errorMessage} (Retry ${request.retryCount}: ${errorMessages.join(', ')})`
+            : errorMessage;
+        this.metadata.errors.push({
+            url: request.url,
+            errorType: errorType,
+            message: fullErrorDetails,
+            timestamp: new Date().toISOString()
+        });
+    }
+    categorizeError(error) {
+        const message = error.message.toLowerCase();
+        if (message.includes('timeout'))
+            return 'timeout';
+        if (message.includes('dns') || message.includes('getaddrinfo'))
+            return 'dns';
+        if (message.includes('connect') || message.includes('econnrefused'))
+            return 'connection';
+        if (message.includes('ssl') || message.includes('certificate'))
+            return 'ssl';
+        if (message.includes('401') || message.includes('403'))
+            return 'auth';
+        if (message.includes('404'))
+            return 'not_found';
+        if (message.includes('429') || message.includes('rate limit'))
+            return 'rate_limit';
+        if (message.includes('500') || message.includes('502') || message.includes('503'))
+            return 'server_error';
+        return 'network';
+    }
+    shouldCrawlUrl(url) {
+        const ext = this.getFileExtension(url);
+        if (ext && this.config.excludeExtensions.includes(ext)) {
+            return false;
+        }
+        if (ext && this.config.includeExtensions.length > 0 &&
+            !this.config.includeExtensions.includes(ext)) {
+            return false;
+        }
+        if (this.config.excludePatterns.length > 0) {
+            if (this.config.excludePatterns.some(pattern => {
+                try {
+                    return new RegExp(pattern).test(url);
+                }
+                catch {
+                    return false;
+                }
+            })) {
+                return false;
+            }
+        }
+        if (this.config.includePatterns.length > 0) {
+            if (!this.config.includePatterns.some(pattern => {
+                try {
+                    return new RegExp(pattern).test(url);
+                }
+                catch {
+                    return false;
+                }
+            })) {
+                return false;
+            }
+        }
+        if (!this.config.crawlExternal && !this.urlManager.isInternal(url)) {
+            return false;
+        }
+        return true;
+    }
+    getFileExtension(url) {
+        try {
+            const pathname = new URL(url).pathname;
+            const parts = pathname.split('.');
+            if (parts.length > 1) {
+                return parts[parts.length - 1].toLowerCase();
+            }
+        }
+        catch { }
+        return '';
+    }
+}
+//# sourceMappingURL=CrawlOrchestrator.js.map

package/build/core/CrawlOrchestrator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CrawlOrchestrator.js","sourceRoot":"","sources":["../../src/core/CrawlOrchestrator.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAsC,GAAG,EAAE,MAAM,SAAS,CAAC;AAC5G,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAM/B,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE1C,MAAM,OAAO,iBAAiB;IAQlB;IACA;IACA;IACA;IACA;IAXF,OAAO,CAAM;IACb,QAAQ,CAAgB;IACxB,UAAU,GAAU,EAAE,CAAC;IACd,gBAAgB,GAAG,GAAG,CAAC;IAChC,aAAa,CAAiB;IAEtC,YACU,MAAmB,EACnB,UAAsB,EACtB,gBAAkC,EAClC,aAA4B,EAC5B,OAAqB;QAJrB,WAAM,GAAN,MAAM,CAAa;QACnB,eAAU,GAAV,UAAU,CAAY;QACtB,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,kBAAa,GAAb,aAAa,CAAe;QAC5B,YAAO,GAAP,OAAO,CAAc;QAG7B,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAE7B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;IAE/C,CAAC;IAEO,qBAAqB;QAC3B,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,MAAM,EAAE,QAAQ;YAChB,SAAS,EAAE,IAAI;YACf,WAAW,EAAE,IAAI;YACjB,QAAQ,EAAE,IAAI;YACd,KAAK,EAAE;gBACL,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,KAAK,EAAE,CAAC;aACT;YACD,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAI7B,MAAM,UAAU,GAAG,qBAAqB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC9D,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,EAAE,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;QAG3E,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC;YACtC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,cAAc,EAAE,KAAK;SACtB,CAAC,CAAC;QAGH,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAGnF,MAAM,UAAU,GAAG;YACjB,MAAM,EAAE,iHAAiH;YACzH,SAAS,EAAE,0EAA0E;SACtF,CAAC;QAEF,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,SAAoC,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC;QAEpG,MAAM,aAAa,GAAG;YACpB,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;YACzC,cAAc,EAAE,EAAE;YAClB,cAAc,EAAE,CAAC;YACjB,iBAAiB,EAAE,CAAC;YACpB,yBAAyB,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI;YACrD,qBAAqB,EAAE,EAAE;YACzB,mBAAmB,EAAE,CAAC,WAAW,EAAE,uBAAuB,CAAC;YAC3D,YAAY;YACZ,kBAAkB,EAAE;gBAClB,KAAK,EAAE,EAAE,OAAO,EAAO,EAAE,EAAE;oBACzB,OAAO,CAAC,OAAO,GAAG;wBAChB,YAAY,EAAE,SAAS;wBACvB,QAAQ,EAAE,4EAA4E;wBACtF,iBAAiB,EAAE,gBAAgB;wBACnC,iBAAiB,EAAE,mBAAmB;wBACtC,YAAY,EAAE,YAAY;wBAC1B,2BAA2B,EAAE,GAAG;qBACjC,CAAC;gBACJ,CAAC;aACF;YACD,cAAc,EAAE,KAAK,EAAE,OAAwB,EAAE,EAAE;gBACjD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,OAAc,CAAC;gBAC5D,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI;oBAAE,OAAO;gBAE/B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAChC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC7E,CAAC;YACD,oBAAoB,EAAE,KAAK,EAAE,OAAwB,EAAE,KAAY,EAAE,EAAE;gBACrE,MAAM,EAAE,OAAO,EAAE,GAAG,OAAc,CAAC;gBACnC,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACjD,CAAC;SACF,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,WAAW,CAAC,aAAoB,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,CAAC;YACH,KAAK,CAAC,oCAAoC,CAAC,CAAC;YAC5C,KAAK,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAG9D,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC/B,KAAK,CAAC,wDAAwD,CAAC,CAAC;YAEhE,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YAChC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAEpC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC;YACjC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACnD,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5D,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAE/C,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YACvD,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAE9C,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC/C,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC/C,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAExC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACjD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;YACvB,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAU,CAAC,CAAC,OAAO,EAAE,CAAC;YAE/D,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrD,IAAI,CAAC,QAAQ,CAAC,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;YAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC;YAC1D,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,GAAG,CAAC;gBACpD,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;gBACjE,CAAC,CAAC,CAAC,CAAC;QAER,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;YAC3D,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;gBACxB,GAAG,EAAE,EAAE;gBACP,SAAS,EAAE,SAAS;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe;gBACzC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YAGT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;oBACjC,KAAK,CAAC,8CAA8C,CAAC,CAAC;gBACxD,CAAC;gBAAC,OAAO,YAAiB,EAAE,CAAC;oBAC3B,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5D,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,GAAW,EACX,IAAY,EACZ,CAAM,EACN,QAAa,EACb,OAAY;QAEZ,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAEjC,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE;YAC3D,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC;YACpC,UAAU,EAAE,QAAQ,CAAC,MAAM,IAAI,GAAG;YAClC,WAAW,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,cAAc,CAAC,IAAI,WAAW;YAC9D,YAAY,EAAE,CAAC;YACf,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC;YAC3C,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,GAAG,CAAC;YAC/C,SAAS,EAAE,EAAE;SACd,EAAE,QAAQ,CAAC,CAAC;QAEb,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEtE,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACnD,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAExC,IAAI,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;oBAExC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;wBAClD,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;oBACvE,CAAC;oBACD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAClC,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAChC,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAChC,CAAC;QACH,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAE1C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;QAC/B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpD,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACjD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACvB,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,kBAAkB,EAAE,CAAC;QAEtE,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,OAAgB,EAAE,KAAY;QAC9D,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,IAAI,eAAe,CAAC;QACtD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;QAElD,MAAM,gBAAgB,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC;YAC/C,CAAC,CAAC,GAAG,YAAY,WAAW,OAAO,CAAC,UAAU,KAAK,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YAC9E,CAAC,CAAC,YAAY,CAAC;QAEjB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;YACxB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,gBAAgB;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,KAAY;QAClC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAE5C,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;QAClD,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7E,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAAE,OAAO,YAAY,CAAC;QACzF,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7E,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC;QACtE,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,WAAW,CAAC;QAChD,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;YAAE,OAAO,YAAY,CAAC;QACnF,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,OAAO,cAAc,CAAC;QAEzG,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,cAAc,CAAC,GAAW;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAGvC,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QAID,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC;YAC/C,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACjD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;gBAC7C,IAAI,CAAC;oBACH,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACvC,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;gBACH,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;gBAC9C,IAAI,CAAC;oBACH,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACvC,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;gBACH,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QACV,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}

package/build/core/CrawlStorage.d.ts

@@ -0,0 +1,33 @@
+import type { CrawlConfig, CrawlMetadata, PageData, LinkData } from '../types/index.js';
+import { CrawlDatabase } from './CrawlDatabase.js';
+export declare class CrawlStorage {
+    private baseDir;
+    private db;
+    private dbPath;
+    private baseUrl?;
+    constructor(outputPath: string, baseUrl?: string);
+    initialize(): Promise<void>;
+    saveConfig(config: CrawlConfig): Promise<void>;
+    loadConfig(): Promise<CrawlConfig>;
+    saveMetadata(metadata: CrawlMetadata, config?: CrawlConfig): Promise<void>;
+    loadMetadata(): Promise<CrawlMetadata | null>;
+    updateMetadata(updates: Partial<CrawlMetadata>): Promise<void>;
+    savePageData(page: PageData): Promise<void>;
+    savePageDataBatch(pages: PageData[]): Promise<void>;
+    loadPageData(url: string): Promise<PageData | null>;
+    loadAllPageData(): Promise<PageData[]>;
+    saveLinkData(links: LinkData[]): Promise<void>;
+    loadLinkData(): Promise<LinkData[]>;
+    appendLinkData(newLinks: LinkData[]): Promise<void>;
+    generateCsvExport(): Promise<void>;
+    exists(): Promise<boolean>;
+    getStats(): Promise<{
+        totalPages: number;
+        totalLinks: number;
+        hasMetadata: boolean;
+        hasConfig: boolean;
+    }>;
+    close(): void;
+    getDatabase(): CrawlDatabase;
+}
+//# sourceMappingURL=CrawlStorage.d.ts.map

package/build/core/CrawlStorage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CrawlStorage.d.ts","sourceRoot":"","sources":["../../src/core/CrawlStorage.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAExF,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,EAAE,CAAiB;IAC3B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAC,CAAS;gBAEb,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM;IAM1C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,UAAU,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9C,UAAU,IAAI,OAAO,CAAC,WAAW,CAAC;IAOlC,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAI1E,YAAY,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAK7C,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAK9D,YAAY,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3C,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAInD,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAInD,eAAe,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKtC,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9C,YAAY,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAInC,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAKnD,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAMlC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAS1B,QAAQ,IAAI,OAAO,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,OAAO,CAAC;QACrB,SAAS,EAAE,OAAO,CAAC;KACpB,CAAC;IAkBF,KAAK,IAAI,IAAI;IAKb,WAAW,IAAI,aAAa;CAG7B"}

package/build/core/CrawlStorage.js

@@ -0,0 +1,94 @@
+import { promises as fs } from 'fs';
+import { join } from 'path';
+import { CrawlConfigSchema } from '../schema/index.js';
+import { CrawlDatabase } from './CrawlDatabase.js';
+export class CrawlStorage {
+    baseDir;
+    db;
+    dbPath;
+    baseUrl;
+    constructor(outputPath, baseUrl) {
+        this.baseDir = outputPath;
+        this.dbPath = join(outputPath, 'crawl-data.db');
+        this.baseUrl = baseUrl;
+    }
+    async initialize() {
+        await fs.mkdir(this.baseDir, { recursive: true });
+        this.db = new CrawlDatabase(this.dbPath);
+    }
+    async saveConfig(config) {
+        const validated = CrawlConfigSchema.parse(config);
+        const filePath = join(this.baseDir, 'config.json');
+        await fs.writeFile(filePath, JSON.stringify(validated, null, 2), 'utf-8');
+    }
+    async loadConfig() {
+        const filePath = join(this.baseDir, 'config.json');
+        const content = await fs.readFile(filePath, 'utf-8');
+        return CrawlConfigSchema.parse(JSON.parse(content));
+    }
+    async saveMetadata(metadata, config) {
+        this.db.saveCrawlMetadata(metadata, this.baseUrl, config);
+    }
+    async loadMetadata() {
+        const config = await this.loadConfig();
+        return this.db.getCrawlMetadata(config.crawlId);
+    }
+    async updateMetadata(updates) {
+        this.db.updateCrawlMetadata(updates);
+    }
+    async savePageData(page) {
+        this.db.savePage(page);
+    }
+    async savePageDataBatch(pages) {
+        this.db.savePageBatch(pages);
+    }
+    async loadPageData(url) {
+        return this.db.getPage(url);
+    }
+    async loadAllPageData() {
+        return this.db.getAllPages();
+    }
+    async saveLinkData(links) {
+        this.db.saveLinks(links);
+    }
+    async loadLinkData() {
+        return this.db.getAllLinks();
+    }
+    async appendLinkData(newLinks) {
+        this.db.saveLinks(newLinks);
+    }
+    async generateCsvExport() {
+        const csvPath = join(this.baseDir, 'crawl-export.csv');
+        this.db.exportToCsv(csvPath);
+    }
+    async exists() {
+        try {
+            await fs.access(this.baseDir);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async getStats() {
+        const [pageCount, linkCount, hasConfig] = await Promise.all([
+            Promise.resolve(this.db.getPageCount()),
+            Promise.resolve(this.db.getLinkCount()),
+            fs.access(join(this.baseDir, 'config.json')).then(() => true).catch(() => false)
+        ]);
+        const metadata = await this.loadMetadata();
+        return {
+            totalPages: pageCount,
+            totalLinks: linkCount,
+            hasMetadata: metadata !== null,
+            hasConfig
+        };
+    }
+    close() {
+        this.db.close();
+    }
+    getDatabase() {
+        return this.db;
+    }
+}
+//# sourceMappingURL=CrawlStorage.js.map

package/build/core/CrawlStorage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CrawlStorage.js","sourceRoot":"","sources":["../../src/core/CrawlStorage.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,OAAO,YAAY;IACf,OAAO,CAAS;IAChB,EAAE,CAAiB;IACnB,MAAM,CAAS;IACf,OAAO,CAAU;IAEzB,YAAY,UAAkB,EAAE,OAAgB;QAC9C,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAElD,IAAI,CAAC,EAAE,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAGD,KAAK,CAAC,UAAU,CAAC,MAAmB;QAClC,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACnD,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACtD,CAAC;IAGD,KAAK,CAAC,YAAY,CAAC,QAAuB,EAAE,MAAoB;QAC9D,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACvC,OAAO,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,OAA+B;QAClD,IAAI,CAAC,EAAE,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAGD,KAAK,CAAC,YAAY,CAAC,IAAc;QAC/B,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,KAAiB;QACvC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAW;QAC5B,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/B,CAAC;IAGD,KAAK,CAAC,YAAY,CAAC,KAAiB;QAClC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,QAAoB;QACvC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAGD,KAAK,CAAC,iBAAiB;QACrB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACvD,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAGD,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QAMZ,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC1D,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;YACvC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;YACvC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC;SACjF,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE3C,OAAO;YACL,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,SAAS;YACrB,WAAW,EAAE,QAAQ,KAAK,IAAI;YAC9B,SAAS;SACV,CAAC;IACJ,CAAC;IAGD,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAGD,WAAW;QACT,OAAO,IAAI,CAAC,EAAE,CAAC;IACjB,CAAC;CACF"}

package/build/core/LinkExtractor.d.ts

@@ -0,0 +1,14 @@
+import type { CheerioAPI } from 'cheerio';
+import type { LinkData } from '../types/index.js';
+export declare class LinkExtractor {
+    private baseDomain;
+    constructor(baseDomain: string);
+    extract($: CheerioAPI, sourceUrl: string, crawlId: string): LinkData[];
+    private shouldSkipLink;
+    private extractAnchorText;
+    private cleanUrl;
+    private normalizeDomain;
+    private isInternal;
+    private detectPlacement;
+}
+//# sourceMappingURL=LinkExtractor.d.ts.map

package/build/core/LinkExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LinkExtractor.d.ts","sourceRoot":"","sources":["../../src/core/LinkExtractor.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAGlD,qBAAa,aAAa;IACxB,OAAO,CAAC,UAAU,CAAS;gBAEf,UAAU,EAAE,MAAM;IAI9B,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE;IAgCtE,OAAO,CAAC,cAAc;IAKtB,OAAO,CAAC,iBAAiB;IAKzB,OAAO,CAAC,QAAQ;IAchB,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,eAAe;CA0BxB"}

package/build/core/LinkExtractor.js

@@ -0,0 +1,91 @@
+import { LinkDataSchema } from '../schema/index.js';
+export class LinkExtractor {
+    baseDomain;
+    constructor(baseDomain) {
+        this.baseDomain = this.normalizeDomain(baseDomain);
+    }
+    extract($, sourceUrl, crawlId) {
+        const links = [];
+        $('a[href]').each((_, el) => {
+            const href = $(el).attr('href')?.trim();
+            if (!href || this.shouldSkipLink(href)) {
+                return;
+            }
+            try {
+                const absolute = new URL(href, sourceUrl);
+                const targetUrl = this.cleanUrl(absolute);
+                const linkData = {
+                    crawlId,
+                    sourceUrl,
+                    targetUrl,
+                    anchorText: this.extractAnchorText($(el)),
+                    isInternal: this.isInternal(targetUrl),
+                    targetDomain: absolute.hostname,
+                    targetStatus: null,
+                    placement: this.detectPlacement($(el)),
+                    discoveredAt: new Date().toISOString()
+                };
+                links.push(LinkDataSchema.parse(linkData));
+            }
+            catch { }
+        });
+        return links;
+    }
+    shouldSkipLink(href) {
+        const skipPrefixes = ['#', 'mailto:', 'tel:', 'javascript:'];
+        return skipPrefixes.some(prefix => href.startsWith(prefix));
+    }
+    extractAnchorText($el) {
+        const text = $el.text().trim();
+        return text.slice(0, 100) || '(no text)';
+    }
+    cleanUrl(url) {
+        let clean = `${url.protocol}//${url.hostname}${url.pathname}`;
+        if (url.search) {
+            clean += url.search;
+        }
+        if (clean.endsWith('/') && clean.length > clean.indexOf('://') + 4) {
+            clean = clean.slice(0, -1);
+        }
+        return clean;
+    }
+    normalizeDomain(domain) {
+        try {
+            const url = domain.startsWith('http') ? domain : `https://${domain}`;
+            return new URL(url).hostname.replace(/^www\./, '');
+        }
+        catch {
+            return domain.replace(/^www\./, '');
+        }
+    }
+    isInternal(url) {
+        try {
+            const urlDomain = new URL(url).hostname.replace(/^www\./, '');
+            return urlDomain === this.baseDomain;
+        }
+        catch {
+            return false;
+        }
+    }
+    detectPlacement($el) {
+        let current = $el.parent();
+        while (current && current.length > 0) {
+            const tagName = current.prop('tagName')?.toLowerCase();
+            const classes = current.attr('class')?.toLowerCase() ?? '';
+            const id = current.attr('id')?.toLowerCase() ?? '';
+            if (tagName === 'footer' || classes.includes('footer') || id.includes('footer')) {
+                return 'footer';
+            }
+            if (tagName === 'nav' || tagName === 'header') {
+                return 'navigation';
+            }
+            const navKeywords = ['nav', 'menu', 'header'];
+            if (navKeywords.some(keyword => classes.includes(keyword) || id.includes(keyword))) {
+                return 'navigation';
+            }
+            current = current.parent();
+        }
+        return 'body';
+    }
+}
+//# sourceMappingURL=LinkExtractor.js.map

package/build/core/LinkExtractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LinkExtractor.js","sourceRoot":"","sources":["../../src/core/LinkExtractor.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,MAAM,OAAO,aAAa;IAChB,UAAU,CAAS;IAE3B,YAAY,UAAkB;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,CAAC,CAAa,EAAE,SAAiB,EAAE,OAAe;QACvD,MAAM,KAAK,GAAe,EAAE,CAAC;QAE7B,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvC,OAAO;YACT,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAE1C,MAAM,QAAQ,GAAa;oBACzB,OAAO;oBACP,SAAS;oBACT,SAAS;oBACT,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBACzC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;oBACtC,YAAY,EAAE,QAAQ,CAAC,QAAQ;oBAC/B,YAAY,EAAE,IAAI;oBAClB,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBACtC,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACvC,CAAC;gBAEF,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;QAC7D,OAAO,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;IAC9D,CAAC;IAEO,iBAAiB,CAAC,GAAQ;QAChC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,WAAW,CAAC;IAC3C,CAAC;IAEO,QAAQ,CAAC,GAAQ;QACvB,IAAI,KAAK,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAE9D,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YACf,KAAK,IAAI,GAAG,CAAC,MAAM,CAAC;QACtB,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACnE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,eAAe,CAAC,MAAc;QACpC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,MAAM,EAAE,CAAC;YACrE,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,GAAW;QAC5B,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAC9D,OAAO,SAAS,KAAK,IAAI,CAAC,UAAU,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,GAAQ;QAC9B,IAAI,OAAO,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QAE3B,OAAO,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC;YACvD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YAC3D,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YAEnD,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAChF,OAAO,QAAQ,CAAC;YAClB,CAAC;YAED,IAAI,OAAO,KAAK,KAAK,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC9C,OAAO,YAAY,CAAC;YACtB,CAAC;YAED,MAAM,WAAW,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC9C,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;gBACnF,OAAO,YAAY,CAAC;YACtB,CAAC;YAED,OAAO,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC7B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/build/core/UrlManager.d.ts

@@ -0,0 +1,21 @@
+export declare class UrlManager {
+    private baseDomain;
+    private discovered;
+    private visited;
+    private sourcePagesMap;
+    constructor(baseDomain: string);
+    normalizeUrl(url: string): string;
+    private normalizeDomain;
+    isInternal(url: string): boolean;
+    addDiscovered(url: string, depth: number, sourceUrl?: string): void;
+    markVisited(url: string): void;
+    isVisited(url: string): boolean;
+    isDiscovered(url: string): boolean;
+    getSourcePages(url: string): string[];
+    getDepth(url: string): number;
+    getTotalDiscovered(): number;
+    getTotalVisited(): number;
+    getMaxDepth(): number;
+    getUnvisitedUrls(): string[];
+}
+//# sourceMappingURL=UrlManager.d.ts.map

package/build/core/UrlManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"UrlManager.d.ts","sourceRoot":"","sources":["../../src/core/UrlManager.ts"],"names":[],"mappings":"AAWA,qBAAa,UAAU;IACrB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,cAAc,CAAuC;gBAEjD,UAAU,EAAE,MAAM;IAI9B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAoBjC,OAAO,CAAC,eAAe;IASvB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAShC,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI;IAgBnE,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAK9B,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI/B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIlC,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE;IAMrC,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAI7B,kBAAkB,IAAI,MAAM;IAI5B,eAAe,IAAI,MAAM;IAIzB,WAAW,IAAI,MAAM;IAIrB,gBAAgB,IAAI,MAAM,EAAE;CAG7B"}