@cosmocoder/mcp-web-docs 2.0.21 → 2.0.23

package/build/crawler/base.d.ts

@@ -1,24 +1,11 @@
 import { CrawlResult } from '../types.js';
 export declare abstract class BaseCrawler {
-    protected seenUrls: Set<string>;
     protected isAborting: boolean;
     private readonly abortController;
     protected readonly abortSignal: AbortSignal;
     private requestCount;
     private lastRequestTime;
-    protected totalUrls: number;
-    protected processedUrls: number;
-    protected onProgress?: (progress: number, description: string) => void;
-    constructor(onProgress?: (progress: number, description: string) => void);
-    protected updateProgress(description: string): void;
-    protected addDiscoveredUrls(count: number): void;
-    protected markUrlProcessed(url: string): void;
     abstract crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
-    protected shouldCrawl(urlString: string): boolean;
-    protected markUrlAsSeen(url: string): void;
-    protected getPathFromUrl(urlString: string): string;
-    protected normalizeUrl(urlString: string): string;
     protected rateLimit(): Promise<void>;
-    protected retryWithBackoff<T>(operation: () => Promise<T>, maxRetries?: number, baseDelay?: number): Promise<T>;
     abort(): void;
 }

package/build/crawler/base.js

@@ -1,105 +1,11 @@
-import { URL } from 'url';
 import { setTimeout as delay } from 'node:timers/promises';
-import { IGNORED_PATHS, RATE_LIMIT } from '../config.js';
-import { logger } from '../util/logger.js';
+import { RATE_LIMIT } from '../config.js';
 export class BaseCrawler {
-    seenUrls;
-    isAborting;
+    isAborting = false;
     abortController = new AbortController();
     abortSignal = this.abortController.signal;
-    requestCount;
-    lastRequestTime;
-    totalUrls;
-    processedUrls;
-    onProgress;
-    constructor(onProgress) {
-        this.seenUrls = new Set();
-        this.isAborting = false;
-        this.requestCount = 0;
-        this.lastRequestTime = 0;
-        this.totalUrls = 1; // Start with 1 for the initial URL
-        this.processedUrls = 0;
-        this.onProgress = onProgress;
-    }
-    updateProgress(description) {
-        if (this.onProgress) {
-            // Calculate progress as percentage (0-100)
-            const progress = Math.min(Math.round((this.processedUrls / this.totalUrls) * 100), 100);
-            this.onProgress(progress, description);
-        }
-    }
-    addDiscoveredUrls(count) {
-        this.totalUrls += count;
-        this.updateProgress('Discovering pages...');
-    }
-    markUrlProcessed(url) {
-        this.processedUrls++;
-        this.markUrlAsSeen(url);
-        this.updateProgress(`Processing page ${this.processedUrls} of ${this.totalUrls}`);
-    }
-    shouldCrawl(urlString) {
-        try {
-            const url = new URL(urlString);
-            // Skip if already seen (using full URL including query params)
-            if (this.seenUrls.has(urlString)) {
-                logger.debug(`[${this.constructor.name}] Skipping already seen URL: ${urlString}`);
-                return false;
-            }
-            // Skip fragments only
-            if (url.hash) {
-                logger.debug(`[${this.constructor.name}] Skipping URL with hash: ${urlString}`);
-                return false;
-            }
-            // Skip non-HTML files only if they have a file extension
-            const ext = url.pathname.split('.').pop()?.toLowerCase();
-            if (ext && ext !== 'html' && ext !== 'htm') {
-                logger.debug(`[${this.constructor.name}] Skipping non-HTML file: ${urlString}`);
-                return false;
-            }
-            // Skip ignored paths only if they match exactly
-            const path = url.pathname.toLowerCase();
-            const isIgnored = IGNORED_PATHS.some((ignored) => {
-                // If ignored path ends with /, treat it as a directory
-                if (ignored.endsWith('/')) {
-                    return path.startsWith(ignored);
-                }
-                // Otherwise match exactly
-                return path === `/${ignored}`;
-            });
-            if (isIgnored) {
-                logger.debug(`[${this.constructor.name}] Skipping ignored path: ${urlString}`);
-                return false;
-            }
-            return true;
-        }
-        catch (error) {
-            logger.debug(`[${this.constructor.name}] Error checking URL: ${urlString}`, error);
-            return false;
-        }
-    }
-    markUrlAsSeen(url) {
-        this.seenUrls.add(url);
-    }
-    getPathFromUrl(urlString) {
-        try {
-            const url = new URL(urlString);
-            return url.pathname + url.search; // Include query params in path
-        }
-        catch {
-            return urlString;
-        }
-    }
-    normalizeUrl(urlString) {
-        try {
-            const url = new URL(urlString);
-            // Remove hash fragment but keep query params
-            url.hash = '';
-            return url.toString().replace(/\/$/, '');
-        }
-        catch {
-            return urlString;
-        }
-    }
+    requestCount = 0;
+    lastRequestTime = 0;
     async rateLimit() {
         const now = Date.now();
         this.requestCount++;
@@ -124,24 +30,6 @@ export class BaseCrawler {
         }
         this.lastRequestTime = Date.now();
     }
-    async retryWithBackoff(operation, maxRetries = 3, baseDelay = 1000) {
-        let lastError;
-        for (let i = 0; i < maxRetries; i++) {
-            try {
-                return await operation();
-            }
-            catch (error) {
-                lastError = error;
-                if (i === maxRetries - 1) {
-                    break;
-                }
-                // Exponential backoff
-                const delay = baseDelay * Math.pow(2, i);
-                await new Promise((resolve) => setTimeout(resolve, delay));
-            }
-        }
-        throw lastError;
-    }
     abort() {
         this.isAborting = true;
         this.abortController.abort();

package/build/crawler/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/crawler/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAEzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAgB,WAAW;IACrB,QAAQ,CAAc;IACtB,UAAU,CAAU;IACb,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrD,YAAY,CAAS;IACrB,eAAe,CAAS;IACtB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,UAAU,CAAmD;IAEvE,YAAY,UAA4D;QACtE,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,mCAAmC;QACvD,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAES,cAAc,CAAC,WAAmB;QAC1C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;YACxF,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAES,iBAAiB,CAAC,KAAa;QACvC,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;QACxB,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,CAAC;IAC9C,CAAC;IAES,gBAAgB,CAAC,GAAW;QACpC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,cAAc,CAAC,mBAAmB,IAAI,CAAC,aAAa,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IACpF,CAAC;IAIS,WAAW,CAAC,SAAiB;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAE/B,+DAA+D;YAC/D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,gCAAgC,SAAS,EAAE,CAAC,CAAC;gBACnF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,sBAAsB;YACtB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,6BAA6B,SAAS,EAAE,CAAC,CAAC;gBAChF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,yDAAyD;YACzD,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;YACzD,IAAI,GAAG,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBAC3C,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,6BAA6B,SAAS,EAAE,CAAC,CAAC;gBAChF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,gDAAgD;YAChD,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC/C,uDAAuD;gBACvD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC1B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClC,CAAC;gBACD,0BAA0B;gBAC1B,OAAO,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC/E,OAAO,KAAK,CAAC;YACf,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,yBAAyB,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;YACnF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAES,aAAa,CAAC,GAAW;QACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAES,cAAc,CAAC,SAAiB;QACxC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,OAAO,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,+BAA+B;QACnE,CAAC;QACD,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAES,YAAY,CAAC,SAAiB;QACtC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,6CAA6C;YAC7C,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAES,KAAK,CAAC,SAAS;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,EAAE,CAAC;QAEpB,0CAA0C;QAC1C,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,UAAU,EAAE,CAAC;YACvD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,0DAA0D;QAC1D,IAAI,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAClC,OAAO;QACT,CAAC;QAED,wCAAwC;QACxC,MAAM,oBAAoB,GAAG,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC;QACxD,IAAI,oBAAoB,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,oBAAoB,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAI,SAA2B,EAAE,aAAqB,CAAC,EAAE,YAAoB,IAAI;QAC/G,IAAI,SAA4B,CAAC;QAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,OAAO,MAAM,SAAS,EAAE,CAAC;YAC3B,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACb,SAAS,GAAG,KAAc,CAAC;gBAC3B,IAAI,CAAC,KAAK,UAAU,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM;gBACR,CAAC;gBAED,sBAAsB;gBACtB,MAAM,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,MAAM,SAAS,CAAC;IAClB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;CACF"}
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/crawler/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,MAAM,OAAgB,WAAW;IACrB,UAAU,GAAG,KAAK,CAAC;IACZ,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrD,YAAY,GAAG,CAAC,CAAC;IACjB,eAAe,GAAG,CAAC,CAAC;IAIlB,KAAK,CAAC,SAAS;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,EAAE,CAAC;QAEpB,0CAA0C;QAC1C,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,UAAU,EAAE,CAAC;YACvD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,0DAA0D;QAC1D,IAAI,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAClC,OAAO;QACT,CAAC;QAED,wCAAwC;QACxC,MAAM,oBAAoB,GAAG,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC;QACxD,IAAI,oBAAoB,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,oBAAoB,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;CACF"}

package/build/crawler/base.test.js

@@ -2,13 +2,8 @@ import { BaseCrawler } from './base.js';
 /**
  * Concrete implementation of BaseCrawler for testing purposes.
  *
- * Since BaseCrawler is abstract, we create this minimal subclass that:
- * 1. Implements the required abstract `crawl` method
- * 2. Exposes protected BaseCrawler methods via public wrappers
- *
- * The wrapper methods (testShouldCrawl, testMarkUrlAsSeen, etc.) simply call
- * the inherited BaseCrawler methods - we're testing the REAL BaseCrawler
- * implementation, not overridden methods.
+ * Since BaseCrawler is abstract, we create this minimal subclass that
+ * implements the required abstract `crawl` method.
  */
 class TestCrawler extends BaseCrawler {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -21,131 +16,21 @@ class TestCrawler extends BaseCrawler {
             title: 'Test',
         };
     }
-    // Expose protected BaseCrawler methods for testing (these call the real implementations)
-    testShouldCrawl(url) {
-        return this.shouldCrawl(url);
-    }
-    testMarkUrlAsSeen(url) {
-        this.markUrlAsSeen(url);
-    }
-    testGetPathFromUrl(url) {
-        return this.getPathFromUrl(url);
-    }
-    testNormalizeUrl(url) {
-        return this.normalizeUrl(url);
-    }
     async testRateLimit() {
         return this.rateLimit();
     }
-    async testRetryWithBackoff(operation, maxRetries) {
-        return this.retryWithBackoff(operation, maxRetries);
-    }
     testAbort() {
         this.abort();
     }
     get isAbortingFlag() {
         return this.isAborting;
     }
-    testUpdateProgress(description) {
-        this.updateProgress(description);
-    }
-    testAddDiscoveredUrls(count) {
-        this.addDiscoveredUrls(count);
-    }
-    testMarkUrlProcessed(url) {
-        this.markUrlProcessed(url);
-    }
 }
 describe('BaseCrawler', () => {
     let crawler;
     beforeEach(() => {
         crawler = new TestCrawler();
     });
-    describe('constructor', () => {
-        it('should accept progress callback', () => {
-            const progressFn = vi.fn();
-            const customCrawler = new TestCrawler(progressFn);
-            customCrawler.testUpdateProgress('Testing');
-            expect(progressFn).toHaveBeenCalled();
-        });
-    });
-    describe('shouldCrawl', () => {
-        it('should return true for valid URLs', () => {
-            // URLs with .html extension pass the file extension check
-            expect(crawler.testShouldCrawl('https://example.com/docs.html')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/api.html')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/page.htm')).toBe(true);
-        });
-        it('should return false for already seen URLs', () => {
-            crawler.testMarkUrlAsSeen('https://example.com/seen');
-            expect(crawler.testShouldCrawl('https://example.com/seen')).toBe(false);
-        });
-        it('should return false for URLs with hash fragments', () => {
-            expect(crawler.testShouldCrawl('https://example.com/page#section')).toBe(false);
-        });
-        it('should return false for non-HTML files', () => {
-            expect(crawler.testShouldCrawl('https://example.com/image.png')).toBe(false);
-            expect(crawler.testShouldCrawl('https://example.com/doc.pdf')).toBe(false);
-            expect(crawler.testShouldCrawl('https://example.com/script.js')).toBe(false);
-            expect(crawler.testShouldCrawl('https://example.com/style.css')).toBe(false);
-        });
-        it('should return true for HTML files', () => {
-            expect(crawler.testShouldCrawl('https://example.com/page.html')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/page.htm')).toBe(true);
-        });
-        it('should return true for HTML file extensions', () => {
-            expect(crawler.testShouldCrawl('https://example.com/docs.html')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/page.htm')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/index.HTML')).toBe(true);
-        });
-        it('should return false for paths without HTML extension', () => {
-            // The implementation's file extension check treats all paths without
-            // .html/.htm extension as non-HTML files
-            expect(crawler.testShouldCrawl('https://example.com/docs')).toBe(false);
-            expect(crawler.testShouldCrawl('https://example.com/')).toBe(false);
-        });
-        it('should return false for invalid URLs', () => {
-            expect(crawler.testShouldCrawl('not-a-url')).toBe(false);
-            expect(crawler.testShouldCrawl('')).toBe(false);
-        });
-    });
-    describe('markUrlAsSeen', () => {
-        it('should mark URL as seen', () => {
-            const url = 'https://example.com/test.html';
-            expect(crawler.testShouldCrawl(url)).toBe(true);
-            crawler.testMarkUrlAsSeen(url);
-            expect(crawler.testShouldCrawl(url)).toBe(false);
-        });
-    });
-    describe('getPathFromUrl', () => {
-        it('should extract pathname from URL', () => {
-            expect(crawler.testGetPathFromUrl('https://example.com/docs/page')).toBe('/docs/page');
-        });
-        it('should include query params in path', () => {
-            expect(crawler.testGetPathFromUrl('https://example.com/search?q=test')).toBe('/search?q=test');
-        });
-        it('should return original string for invalid URL', () => {
-            expect(crawler.testGetPathFromUrl('not-a-url')).toBe('not-a-url');
-        });
-        it('should handle root path', () => {
-            expect(crawler.testGetPathFromUrl('https://example.com')).toBe('/');
-            expect(crawler.testGetPathFromUrl('https://example.com/')).toBe('/');
-        });
-    });
-    describe('normalizeUrl', () => {
-        it('should remove hash fragments', () => {
-            expect(crawler.testNormalizeUrl('https://example.com/page#section')).toBe('https://example.com/page');
-        });
-        it('should remove trailing slash', () => {
-            expect(crawler.testNormalizeUrl('https://example.com/docs/')).toBe('https://example.com/docs');
-        });
-        it('should preserve query params', () => {
-            expect(crawler.testNormalizeUrl('https://example.com/search?q=test')).toBe('https://example.com/search?q=test');
-        });
-        it('should return original string for invalid URL', () => {
-            expect(crawler.testNormalizeUrl('not-a-url')).toBe('not-a-url');
-        });
-    });
     describe('abort', () => {
         it('should set isAborting flag to true', () => {
             expect(crawler.isAbortingFlag).toBe(false);
@@ -153,61 +38,6 @@ describe('BaseCrawler', () => {
             expect(crawler.isAbortingFlag).toBe(true);
         });
     });
-    describe('retryWithBackoff', () => {
-        it('should return result on successful operation', async () => {
-            const operation = vi.fn().mockResolvedValue('success');
-            const result = await crawler.testRetryWithBackoff(operation);
-            expect(result).toBe('success');
-            expect(operation).toHaveBeenCalledTimes(1);
-        });
-        it('should retry on failure', async () => {
-            const operation = vi.fn().mockRejectedValueOnce(new Error('fail')).mockResolvedValue('success');
-            vi.spyOn(global, 'setTimeout').mockImplementation((callback) => {
-                callback();
-                return 0;
-            });
-            const result = await crawler.testRetryWithBackoff(operation);
-            expect(result).toBe('success');
-            expect(operation).toHaveBeenCalledTimes(2);
-            vi.restoreAllMocks();
-        });
-        it('should throw after max retries', async () => {
-            const operation = vi.fn().mockRejectedValue(new Error('persistent error'));
-            vi.spyOn(global, 'setTimeout').mockImplementation((callback) => {
-                callback();
-                return 0;
-            });
-            await expect(crawler.testRetryWithBackoff(operation, 3)).rejects.toThrow('persistent error');
-            expect(operation).toHaveBeenCalledTimes(3);
-            vi.restoreAllMocks();
-        });
-    });
-    describe('progress tracking', () => {
-        it('should call progress callback with updateProgress', () => {
-            const progressFn = vi.fn();
-            const progressCrawler = new TestCrawler(progressFn);
-            progressCrawler.testUpdateProgress('Processing...');
-            expect(progressFn).toHaveBeenCalledWith(expect.any(Number), 'Processing...');
-        });
-        it('should track discovered URLs', () => {
-            const progressFn = vi.fn();
-            const progressCrawler = new TestCrawler(progressFn);
-            progressCrawler.testAddDiscoveredUrls(5);
-            expect(progressFn).toHaveBeenCalled();
-        });
-        it('should track processed URLs', () => {
-            const progressFn = vi.fn();
-            const progressCrawler = new TestCrawler(progressFn);
-            progressCrawler.testMarkUrlProcessed('https://example.com/page1');
-            expect(progressFn).toHaveBeenCalled();
-        });
-        it('should not throw when progress callback not provided', () => {
-            const noCrawler = new TestCrawler();
-            expect(() => noCrawler.testUpdateProgress('Test')).not.toThrow();
-            expect(() => noCrawler.testAddDiscoveredUrls(5)).not.toThrow();
-            expect(() => noCrawler.testMarkUrlProcessed('https://example.com')).not.toThrow();
-        });
-    });
     describe('crawl', () => {
         it('should yield CrawlResult', async () => {
             const results = [];

package/build/crawler/base.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.test.js","sourceRoot":"","sources":["../../src/crawler/base.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC;;;;;;;;;;GAUG;AACH,MAAM,WAAY,SAAQ,WAAW;IACnC,6DAA6D;IAC7D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAY;QACvB,MAAM;YACJ,GAAG,EAAE,qBAAqB;YAC1B,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,eAAe;YACxB,aAAa,EAAE,MAAM;YACrB,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IAED,yFAAyF;IAClF,eAAe,CAAC,GAAW;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAEM,iBAAiB,CAAC,GAAW;QAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CAAC,GAAW;QACnC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAEM,gBAAgB,CAAC,GAAW;QACjC,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAAI,SAA2B,EAAE,UAAmB;QACnF,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACtD,CAAC;IAEM,SAAS;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,IAAW,cAAc;QACvB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAEM,kBAAkB,CAAC,WAAmB;QAC3C,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAEM,qBAAqB,CAAC,KAAa;QACxC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAEM,oBAAoB,CAAC,GAAW;QACrC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;CACF;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAClD,aAAa,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,0DAA0D;YAC1D,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,OAAO,CAAC,iBAAiB,CAAC,0BAA0B,CAAC,CAAC;YACtD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,6BAA6B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,gCAAgC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,qEAAqE;YACrE,yCAAyC;YACzC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxE,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,GAAG,GAAG,+BAA+B,CAAC;YAC5C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChD,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,mCAAmC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpE,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,mCAAmC,CAAC,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAClH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3C,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEhG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,QAAoB,EAAE,EAAE;gBACzE,QAAQ,EAAE,CAAC;gBACX,OAAO,CAA8B,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE3C,EAAE,CAAC,eAAe,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;YAE3E,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,QAAoB,EAAE,EAAE;gBACzE,QAAQ,EAAE,CAAC;gBACX,OAAO,CAA8B,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC7F,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE3C,EAAE,CAAC,eAAe,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAEpD,eAAe,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;YAEpD,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAEpD,eAAe,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAEzC,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAEpD,eAAe,CAAC,oBAAoB,CAAC,2BAA2B,CAAC,CAAC;YAElE,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,SAAS,GAAG,IAAI,WAAW,EAAE,CAAC;YAEpC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACjE,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAC/D,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,oBAAoB,CAAC,qBAAqB,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACpF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"base.test.js","sourceRoot":"","sources":["../../src/crawler/base.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC;;;;;GAKG;AACH,MAAM,WAAY,SAAQ,WAAW;IACnC,6DAA6D;IAC7D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAY;QACvB,MAAM;YACJ,GAAG,EAAE,qBAAqB;YAC1B,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,eAAe;YACxB,aAAa,EAAE,MAAM;YACrB,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAEM,SAAS;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,IAAW,cAAc;QACvB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;CACF;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3C,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/content-utils.d.ts

@@ -1,2 +1 @@
 export declare function cleanContent(text: string): string;
-export declare function cleanCodeBlock(code: string): string;

package/build/crawler/content-utils.js

@@ -11,12 +11,4 @@ export function cleanContent(text) {
         .replace(/\n{3,}/g, '\n\n') // Max 2 consecutive newlines
         .trim();
 }
-// Utility function specifically for cleaning code blocks
-export function cleanCodeBlock(code) {
-    return code
-        .replace(/^\s+|\s+$/g, '') // Trim whitespace
-        .replace(/\t/g, '  ') // Convert tabs to spaces
-        .replace(/\n{3,}/g, '\n\n') // Reduce multiple blank lines
-        .replace(/\u00A0/g, ' '); // Replace non-breaking spaces
-}
 //# sourceMappingURL=content-utils.js.map

package/build/crawler/content-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"content-utils.js","sourceRoot":"","sources":["../../src/crawler/content-utils.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,2BAA2B;SACjD,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,yBAAyB;SAChD,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,yBAAyB;SAC9C,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,8DAA8D;SACxF,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,sDAAsD;SACpF,IAAI,CAAC,IAAI,CAAC;SACV,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,6BAA6B;SACxD,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI;SACR,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,kBAAkB;SAC5C,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,yBAAyB;SAC9C,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,8BAA8B;SACzD,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,8BAA8B;AAC5D,CAAC"}
+{"version":3,"file":"content-utils.js","sourceRoot":"","sources":["../../src/crawler/content-utils.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,2BAA2B;SACjD,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,yBAAyB;SAChD,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,yBAAyB;SAC9C,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,8DAA8D;SACxF,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,sDAAsD;SACpF,IAAI,CAAC,IAAI,CAAC;SACV,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,6BAA6B;SACxD,IAAI,EAAE,CAAC;AACZ,CAAC"}

package/build/crawler/content-utils.test.js

@@ -1,4 +1,4 @@
-import { cleanContent, cleanCodeBlock } from './content-utils.js';
+import { cleanContent } from './content-utils.js';
 describe('Content Utilities', () => {
     describe('cleanContent', () => {
         it('should convert escaped newlines to actual newlines', () => {
@@ -55,45 +55,5 @@ describe('Content Utilities', () => {
             expect(result).toBe('Title\n\n Content with multiple spaces\n Tabbed');
         });
     });
-    describe('cleanCodeBlock', () => {
-        it('should trim leading and trailing whitespace', () => {
-            const input = '   \nconst x = 1;\n   ';
-            const result = cleanCodeBlock(input);
-            expect(result).toBe('const x = 1;');
-        });
-        it('should convert tabs to spaces', () => {
-            const input = 'function test() {\n\treturn true;\n}';
-            const result = cleanCodeBlock(input);
-            expect(result).toBe('function test() {\n  return true;\n}');
-        });
-        it('should reduce multiple blank lines', () => {
-            const input = 'line 1\n\n\n\nline 2';
-            const result = cleanCodeBlock(input);
-            expect(result).toBe('line 1\n\nline 2');
-        });
-        it('should replace non-breaking spaces with regular spaces', () => {
-            const input = 'const\u00A0x\u00A0=\u00A01;';
-            const result = cleanCodeBlock(input);
-            expect(result).toBe('const x = 1;');
-        });
-        it('should handle empty string', () => {
-            expect(cleanCodeBlock('')).toBe('');
-        });
-        it('should handle code with multiple formatting issues', () => {
-            const input = '\n\t\tconst x\u00A0= 1;\n\n\n\n\treturn x;\n';
-            const result = cleanCodeBlock(input);
-            expect(result).toBe('const x = 1;\n\n  return x;');
-        });
-        it('should preserve single blank lines', () => {
-            const input = 'line 1\n\nline 2';
-            const result = cleanCodeBlock(input);
-            expect(result).toBe('line 1\n\nline 2');
-        });
-        it('should handle code with mixed indentation', () => {
-            const input = 'function test() {\n\tif (true) {\n\t\treturn;\n\t}\n}';
-            const result = cleanCodeBlock(input);
-            expect(result).toBe('function test() {\n  if (true) {\n    return;\n  }\n}');
-        });
-    });
 });
 //# sourceMappingURL=content-utils.test.js.map

package/build/crawler/content-utils.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"content-utils.test.js","sourceRoot":"","sources":["../../src/crawler/content-utils.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAElE,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;YAC5D,MAAM,KAAK,GAAG,0BAA0B,CAAC;YACzC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,KAAK,GAAG,4BAA4B,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,KAAK,GAAG,kBAAkB,CAAC;YACjC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,2EAA2E;YAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,KAAK,GAAG,wBAAwB,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,KAAK,GAAG,sBAAsB,CAAC;YACrC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,KAAK,GAAG,oCAAoC,CAAC;YACnD,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,KAAK,GAAG,wBAAwB,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,KAAK,GAAG,4BAA4B,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,KAAK,GAAG,0EAA0E,CAAC;YACzF,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,oFAAoF;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,KAAK,GAAG,wBAAwB,CAAC;YACvC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,KAAK,GAAG,sCAAsC,CAAC;YACrD,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,KAAK,GAAG,sBAAsB,CAAC;YACrC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;YAChE,MAAM,KAAK,GAAG,6BAA6B,CAAC;YAC5C,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;YAC5D,MAAM,KAAK,GAAG,8CAA8C,CAAC;YAC7D,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,KAAK,GAAG,kBAAkB,CAAC;YACjC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,KAAK,GAAG,uDAAuD,CAAC;YACtE,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"content-utils.test.js","sourceRoot":"","sources":["../../src/crawler/content-utils.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;YAC5D,MAAM,KAAK,GAAG,0BAA0B,CAAC;YACzC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,KAAK,GAAG,4BAA4B,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,KAAK,GAAG,kBAAkB,CAAC;YACjC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,2EAA2E;YAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,KAAK,GAAG,wBAAwB,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,KAAK,GAAG,sBAAsB,CAAC;YACrC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,KAAK,GAAG,oCAAoC,CAAC;YACnD,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,KAAK,GAAG,wBAAwB,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,KAAK,GAAG,4BAA4B,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,KAAK,GAAG,0EAA0E,CAAC;YACzF,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,oFAAoF;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/crawlee-crawler.d.ts

@@ -1,25 +1,6 @@
 import { CrawlResult } from '../types.js';
 import { BaseCrawler } from './base.js';
-/** Storage state for authentication (cookies and localStorage) */
-export interface StorageState {
-    cookies: Array<{
-        name: string;
-        value: string;
-        domain: string;
-        path: string;
-        expires?: number;
-        httpOnly?: boolean;
-        secure?: boolean;
-        sameSite?: 'Strict' | 'Lax' | 'None';
-    }>;
-    origins?: Array<{
-        origin: string;
-        localStorage: Array<{
-            name: string;
-            value: string;
-        }>;
-    }>;
-}
+import { type ValidatedStorageState } from '../util/security.js';
 export declare class CrawleeCrawler extends BaseCrawler {
     private crawler;
     private queueManager;
@@ -39,7 +20,7 @@ export declare class CrawleeCrawler extends BaseCrawler {
     /**
      * Set authentication cookies/localStorage to use when crawling
      */
-    setStorageState(state: StorageState): void;
+    setStorageState(state: ValidatedStorageState): void;
     /**
      * Check if a URL is within the allowed domain for this crawl.
      * This prevents following redirects or links to external domains.

package/build/crawler/crawlee-crawler.js

@@ -500,7 +500,6 @@ export class CrawleeCrawler extends BaseCrawler {
                                 return;
                             }
                             this.queueManager.addResult(result);
-                            this.markUrlProcessed(request.url);
                             break;
                         }
                     }