@cosmocoder/mcp-web-docs 2.0.20 → 2.0.22

package/build/config.d.ts

@@ -1,36 +1,17 @@
 export interface DocsConfig {
     githubToken?: string;
-    maxDepth: number;
-    maxRequestsPerCrawl: number;
     maxChunkSize: number;
-    maxConcurrentRequests: number;
     cacheSize: number;
     dataDir: string;
     dbPath: string;
     vectorDbPath: string;
 }
 export declare function loadConfig(): Promise<DocsConfig>;
-export declare const IGNORED_PATHS: string[];
 export declare const RATE_LIMIT: {
     maxRequests: number;
     timeWindow: number;
     minDelay: number;
 };
-export declare const QUEUE_OPTIONS: {
-    maxRequestRetries: number;
-    retryDelay: number;
-    maxRequestsPerCrawl: number;
-};
-export declare const GITHUB_RATE_LIMIT: {
-    unauthenticated: {
-        maxRequests: number;
-        timeWindow: number;
-    };
-    authenticated: {
-        maxRequests: number;
-        timeWindow: number;
-    };
-};
 export declare function isValidUrl(urlString: string): boolean;
 /**
  * Validate URL and check for SSRF attacks (blocks private/internal networks)

package/build/config.js

@@ -5,10 +5,7 @@ import { logger } from './util/logger.js';
 import { validatePublicUrl } from './util/security.js';
 const DATA_DIR = join(homedir(), '.mcp-web-docs');
 const DEFAULT_CONFIG = {
-    maxDepth: 4,
-    maxRequestsPerCrawl: 1000, // Match DocsCrawler default for better coverage
     maxChunkSize: 1000,
-    maxConcurrentRequests: 3, // Allow concurrent requests for better performance while maintaining stability
     cacheSize: 1000,
     dataDir: DATA_DIR,
     dbPath: join(DATA_DIR, 'docs.db'),
@@ -44,59 +41,12 @@ export async function loadConfig() {
     });
     return config;
 }
-// Constants for indexing
-export const IGNORED_PATHS = [
-    'favicon.ico',
-    'robots.txt',
-    '.rst.txt',
-    'genindex',
-    'py-modindex',
-    'search.html',
-    'search',
-    'genindex.html',
-    'changelog',
-    'changelog.html',
-    'assets/',
-    'static/',
-    'images/',
-    'img/',
-    'css/',
-    'js/',
-    'fonts/',
-    // Common repository paths to ignore
-    'node_modules/',
-    'vendor/',
-    'test/',
-    'tests/',
-    'example/',
-    'examples/',
-    'build/',
-    'dist/',
-    '.git/',
-];
 // Rate limiting constants
 export const RATE_LIMIT = {
     maxRequests: 60, // Increased for better throughput
     timeWindow: 60 * 1000, // 1 minute
     minDelay: 250, // Reduced delay between requests
 };
-// Queue configuration
-export const QUEUE_OPTIONS = {
-    maxRequestRetries: 2,
-    retryDelay: 1000,
-    maxRequestsPerCrawl: 2000, // Increased from default 1000
-};
-// GitHub API rate limits
-export const GITHUB_RATE_LIMIT = {
-    unauthenticated: {
-        maxRequests: 60,
-        timeWindow: 60 * 60 * 1000, // 1 hour
-    },
-    authenticated: {
-        maxRequests: 5000,
-        timeWindow: 60 * 60 * 1000, // 1 hour
-    },
-};
 // Utility function to validate URLs
 export function isValidUrl(urlString) {
     try {

package/build/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAcvD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAElD,MAAM,cAAc,GAAe;IACjC,QAAQ,EAAE,CAAC;IACX,mBAAmB,EAAE,IAAI,EAAE,gDAAgD;IAC3E,YAAY,EAAE,IAAI;IAClB,qBAAqB,EAAE,CAAC,EAAE,+EAA+E;IACzG,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,QAAQ;IACjB,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;IACjC,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;CACxC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAE/C,+CAA+C;IAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAC7C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC9C,CAAC;IAED,0DAA0D;IAC1D,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,qCAAqC,QAAQ,EAAE,CAAC,CAAC;QAC9D,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3C,gCAAgC;QAChC,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,mBAAmB,GAAG,iBAAiB,CAAC;QACpD,MAAM,KAAK,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAe;QACzB,GAAG,cAAc;QACjB,WAAW;KACZ,CAAC;IAEF,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;QAC7C,GAAG,MAAM;QACT,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KAC7C,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yBAAyB;AACzB,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,aAAa;IACb,YAAY;IACZ,UAAU;IACV,UAAU;IACV,aAAa;IACb,aAAa;IACb,QAAQ;IACR,eAAe;IACf,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,SAAS;IACT,SAAS;IACT,MAAM;IACN,MAAM;IACN,KAAK;IACL,QAAQ;IACR,oCAAoC;IACpC,eAAe;IACf,SAAS;IACT,OAAO;IACP,QAAQ;IACR,UAAU;IACV,WAAW;IACX,QAAQ;IACR,OAAO;IACP,OAAO;CACR,CAAC;AAEF,0BAA0B;AAC1B,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,WAAW,EAAE,EAAE,EAAE,kCAAkC;IACnD,UAAU,EAAE,EAAE,GAAG,IAAI,EAAE,WAAW;IAClC,QAAQ,EAAE,GAAG,EAAE,iCAAiC;CACjD,CAAC;AAEF,sBAAsB;AACtB,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,iBAAiB,EAAE,CAAC;IACpB,UAAU,EAAE,IAAI;IAChB,mBAAmB,EAAE,IAAI,EAAE,8BAA8B;CAC1D,CAAC;AAEF,yBAAyB;AACzB,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,eAAe,EAAE;QACf,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,SAAS;KACtC;IACD,aAAa,EAAE;QACb,WAAW,EAAE,IAAI;QACjB,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,SAAS;KACtC;CACF,CAAC;AAEF,oCAAoC;AACpC,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC/D,CAAC;IACD,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAAiB,EAAE,YAAY,GAAG,KAAK;IACtE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,4CAA4C,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC5C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,wBAAwB;QACxB,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,CAAC;QACL,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,WAAW,CAAC,SAAiB;IAC3C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC;IACvC,CAAC;IACD,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAWvD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAElD,MAAM,cAAc,GAAe;IACjC,YAAY,EAAE,IAAI;IAClB,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,QAAQ;IACjB,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;IACjC,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;CACxC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAE/C,+CAA+C;IAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAC7C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC9C,CAAC;IAED,0DAA0D;IAC1D,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,qCAAqC,QAAQ,EAAE,CAAC,CAAC;QAC9D,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3C,gCAAgC;QAChC,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,mBAAmB,GAAG,iBAAiB,CAAC;QACpD,MAAM,KAAK,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAe;QACzB,GAAG,cAAc;QACjB,WAAW;KACZ,CAAC;IAEF,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;QAC7C,GAAG,MAAM;QACT,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KAC7C,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,WAAW,EAAE,EAAE,EAAE,kCAAkC;IACnD,UAAU,EAAE,EAAE,GAAG,IAAI,EAAE,WAAW;IAClC,QAAQ,EAAE,GAAG,EAAE,iCAAiC;CACjD,CAAC;AAEF,oCAAoC;AACpC,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC/D,CAAC;IACD,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAAiB,EAAE,YAAY,GAAG,KAAK;IACtE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,4CAA4C,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC5C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,wBAAwB;QACxB,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,CAAC;QACL,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,WAAW,CAAC,SAAiB;IAC3C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC;IACvC,CAAC;IACD,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/build/config.test.js

@@ -1,4 +1,4 @@
-import { isValidUrl, isValidPublicUrl, normalizeUrl, isGitHubUrl, IGNORED_PATHS, RATE_LIMIT, QUEUE_OPTIONS, GITHUB_RATE_LIMIT, } from './config.js';
+import { isValidUrl, isValidPublicUrl, normalizeUrl, isGitHubUrl, RATE_LIMIT } from './config.js';
 describe('Configuration Utilities', () => {
     describe('isValidUrl', () => {
         it('should accept valid HTTP URLs', () => {
@@ -99,20 +99,6 @@ describe('Configuration Utilities', () => {
         });
     });
     describe('Constants', () => {
-        describe('IGNORED_PATHS', () => {
-            it('should contain common non-content paths', () => {
-                expect(IGNORED_PATHS).toContain('favicon.ico');
-                expect(IGNORED_PATHS).toContain('robots.txt');
-                expect(IGNORED_PATHS).toContain('assets/');
-                expect(IGNORED_PATHS).toContain('node_modules/');
-            });
-            it('should be an array of strings', () => {
-                expect(Array.isArray(IGNORED_PATHS)).toBe(true);
-                IGNORED_PATHS.forEach((path) => {
-                    expect(typeof path).toBe('string');
-                });
-            });
-        });
         describe('RATE_LIMIT', () => {
             it('should have reasonable default values', () => {
                 expect(RATE_LIMIT.maxRequests).toBeGreaterThan(0);
@@ -120,22 +106,6 @@ describe('Configuration Utilities', () => {
                 expect(RATE_LIMIT.minDelay).toBeGreaterThanOrEqual(0);
             });
         });
-        describe('QUEUE_OPTIONS', () => {
-            it('should have retry configuration', () => {
-                expect(QUEUE_OPTIONS.maxRequestRetries).toBeGreaterThanOrEqual(0);
-                expect(QUEUE_OPTIONS.retryDelay).toBeGreaterThan(0);
-                expect(QUEUE_OPTIONS.maxRequestsPerCrawl).toBeGreaterThan(0);
-            });
-        });
-        describe('GITHUB_RATE_LIMIT', () => {
-            it('should have different limits for authenticated vs unauthenticated', () => {
-                expect(GITHUB_RATE_LIMIT.authenticated.maxRequests).toBeGreaterThan(GITHUB_RATE_LIMIT.unauthenticated.maxRequests);
-            });
-            it('should have time windows defined', () => {
-                expect(GITHUB_RATE_LIMIT.authenticated.timeWindow).toBeGreaterThan(0);
-                expect(GITHUB_RATE_LIMIT.unauthenticated.timeWindow).toBeGreaterThan(0);
-            });
-        });
     });
 });
 //# sourceMappingURL=config.test.js.map

package/build/config.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.test.js","sourceRoot":"","sources":["../src/config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,aAAa,EACb,UAAU,EACV,aAAa,EACb,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,MAAM,CAAC,UAAU,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjE,MAAM,CAAC,UAAU,CAAC,oCAAoC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClE,MAAM,CAAC,gBAAgB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9D,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,CAAC,gBAAgB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/D,MAAM,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACzE,MAAM,CAAC,YAAY,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClG,MAAM,CAAC,YAAY,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACxE,MAAM,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAC/D,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,WAAW,CAAC,kDAAkD,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnF,MAAM,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChE,MAAM,CAAC,WAAW,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1D,MAAM,CAAC,WAAW,CAAC,uDAAuD,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;YAC7B,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;gBACjD,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBAC/C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;gBAC9C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAC3C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YACnD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;gBACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChD,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC7B,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACrC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;YAC1B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;gBAC/C,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBACjD,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;YAC7B,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;gBACzC,MAAM,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;gBAClE,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBACpD,MAAM,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;YACjC,EAAE,CAAC,mEAAmE,EAAE,GAAG,EAAE;gBAC3E,MAAM,CAAC,iBAAiB,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,iBAAiB,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;YACrH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;gBAC1C,MAAM,CAAC,iBAAiB,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBACtE,MAAM,CAAC,iBAAiB,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"config.test.js","sourceRoot":"","sources":["../src/config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAElG,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,MAAM,CAAC,UAAU,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjE,MAAM,CAAC,UAAU,CAAC,oCAAoC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClE,MAAM,CAAC,gBAAgB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9D,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,CAAC,gBAAgB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/D,MAAM,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACzE,MAAM,CAAC,YAAY,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClG,MAAM,CAAC,YAAY,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACxE,MAAM,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAC/D,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,WAAW,CAAC,kDAAkD,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnF,MAAM,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChE,MAAM,CAAC,WAAW,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1D,MAAM,CAAC,WAAW,CAAC,uDAAuD,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;YAC1B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;gBAC/C,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBACjD,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/base.d.ts

@@ -1,26 +1,11 @@
 import { CrawlResult } from '../types.js';
 export declare abstract class BaseCrawler {
-    protected readonly maxDepth: number;
-    protected readonly maxRequestsPerCrawl: number;
-    protected seenUrls: Set<string>;
     protected isAborting: boolean;
     private readonly abortController;
     protected readonly abortSignal: AbortSignal;
     private requestCount;
     private lastRequestTime;
-    protected totalUrls: number;
-    protected processedUrls: number;
-    protected onProgress?: (progress: number, description: string) => void;
-    constructor(maxDepth?: number, maxRequestsPerCrawl?: number, onProgress?: (progress: number, description: string) => void);
-    protected updateProgress(description: string): void;
-    protected addDiscoveredUrls(count: number): void;
-    protected markUrlProcessed(url: string): void;
-    abstract crawl(url: string, maxDepth?: number): AsyncGenerator<CrawlResult, void, unknown>;
-    protected shouldCrawl(urlString: string): boolean;
-    protected markUrlAsSeen(url: string): void;
-    protected getPathFromUrl(urlString: string): string;
-    protected normalizeUrl(urlString: string): string;
+    abstract crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
     protected rateLimit(): Promise<void>;
-    protected retryWithBackoff<T>(operation: () => Promise<T>, maxRetries?: number, baseDelay?: number): Promise<T>;
     abort(): void;
 }

package/build/crawler/base.js

@@ -1,109 +1,11 @@
-import { URL } from 'url';
 import { setTimeout as delay } from 'node:timers/promises';
-import { IGNORED_PATHS, RATE_LIMIT } from '../config.js';
-import { logger } from '../util/logger.js';
+import { RATE_LIMIT } from '../config.js';
 export class BaseCrawler {
-    maxDepth;
-    maxRequestsPerCrawl;
-    seenUrls;
-    isAborting;
+    isAborting = false;
     abortController = new AbortController();
     abortSignal = this.abortController.signal;
-    requestCount;
-    lastRequestTime;
-    totalUrls;
-    processedUrls;
-    onProgress;
-    constructor(maxDepth = 4, maxRequestsPerCrawl = 1000, onProgress) {
-        this.maxDepth = maxDepth;
-        this.maxRequestsPerCrawl = maxRequestsPerCrawl;
-        this.seenUrls = new Set();
-        this.isAborting = false;
-        this.requestCount = 0;
-        this.lastRequestTime = 0;
-        this.totalUrls = 1; // Start with 1 for the initial URL
-        this.processedUrls = 0;
-        this.onProgress = onProgress;
-    }
-    updateProgress(description) {
-        if (this.onProgress) {
-            // Calculate progress as percentage (0-100)
-            const progress = Math.min(Math.round((this.processedUrls / this.totalUrls) * 100), 100);
-            this.onProgress(progress, description);
-        }
-    }
-    addDiscoveredUrls(count) {
-        this.totalUrls += count;
-        this.updateProgress('Discovering pages...');
-    }
-    markUrlProcessed(url) {
-        this.processedUrls++;
-        this.markUrlAsSeen(url);
-        this.updateProgress(`Processing page ${this.processedUrls} of ${this.totalUrls}`);
-    }
-    shouldCrawl(urlString) {
-        try {
-            const url = new URL(urlString);
-            // Skip if already seen (using full URL including query params)
-            if (this.seenUrls.has(urlString)) {
-                logger.debug(`[${this.constructor.name}] Skipping already seen URL: ${urlString}`);
-                return false;
-            }
-            // Skip fragments only
-            if (url.hash) {
-                logger.debug(`[${this.constructor.name}] Skipping URL with hash: ${urlString}`);
-                return false;
-            }
-            // Skip non-HTML files only if they have a file extension
-            const ext = url.pathname.split('.').pop()?.toLowerCase();
-            if (ext && ext !== 'html' && ext !== 'htm') {
-                logger.debug(`[${this.constructor.name}] Skipping non-HTML file: ${urlString}`);
-                return false;
-            }
-            // Skip ignored paths only if they match exactly
-            const path = url.pathname.toLowerCase();
-            const isIgnored = IGNORED_PATHS.some((ignored) => {
-                // If ignored path ends with /, treat it as a directory
-                if (ignored.endsWith('/')) {
-                    return path.startsWith(ignored);
-                }
-                // Otherwise match exactly
-                return path === `/${ignored}`;
-            });
-            if (isIgnored) {
-                logger.debug(`[${this.constructor.name}] Skipping ignored path: ${urlString}`);
-                return false;
-            }
-            return true;
-        }
-        catch (error) {
-            logger.debug(`[${this.constructor.name}] Error checking URL: ${urlString}`, error);
-            return false;
-        }
-    }
-    markUrlAsSeen(url) {
-        this.seenUrls.add(url);
-    }
-    getPathFromUrl(urlString) {
-        try {
-            const url = new URL(urlString);
-            return url.pathname + url.search; // Include query params in path
-        }
-        catch {
-            return urlString;
-        }
-    }
-    normalizeUrl(urlString) {
-        try {
-            const url = new URL(urlString);
-            // Remove hash fragment but keep query params
-            url.hash = '';
-            return url.toString().replace(/\/$/, '');
-        }
-        catch {
-            return urlString;
-        }
-    }
+    requestCount = 0;
+    lastRequestTime = 0;
     async rateLimit() {
         const now = Date.now();
         this.requestCount++;
@@ -128,24 +30,6 @@ export class BaseCrawler {
         }
         this.lastRequestTime = Date.now();
     }
-    async retryWithBackoff(operation, maxRetries = 3, baseDelay = 1000) {
-        let lastError;
-        for (let i = 0; i < maxRetries; i++) {
-            try {
-                return await operation();
-            }
-            catch (error) {
-                lastError = error;
-                if (i === maxRetries - 1) {
-                    break;
-                }
-                // Exponential backoff
-                const delay = baseDelay * Math.pow(2, i);
-                await new Promise((resolve) => setTimeout(resolve, delay));
-            }
-        }
-        throw lastError;
-    }
     abort() {
         this.isAborting = true;
         this.abortController.abort();

package/build/crawler/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/crawler/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAEzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAgB,WAAW;IAYV;IACA;IAZX,QAAQ,CAAc;IACtB,UAAU,CAAU;IACb,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrD,YAAY,CAAS;IACrB,eAAe,CAAS;IACtB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,UAAU,CAAmD;IAEvE,YACqB,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EACrD,UAA4D;QAFzC,aAAQ,GAAR,QAAQ,CAAY;QACpB,wBAAmB,GAAnB,mBAAmB,CAAe;QAGrD,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,mCAAmC;QACvD,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAES,cAAc,CAAC,WAAmB;QAC1C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;YACxF,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAES,iBAAiB,CAAC,KAAa;QACvC,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;QACxB,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,CAAC;IAC9C,CAAC;IAES,gBAAgB,CAAC,GAAW;QACpC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,cAAc,CAAC,mBAAmB,IAAI,CAAC,aAAa,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IACpF,CAAC;IAIS,WAAW,CAAC,SAAiB;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAE/B,+DAA+D;YAC/D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,gCAAgC,SAAS,EAAE,CAAC,CAAC;gBACnF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,sBAAsB;YACtB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,6BAA6B,SAAS,EAAE,CAAC,CAAC;gBAChF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,yDAAyD;YACzD,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;YACzD,IAAI,GAAG,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBAC3C,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,6BAA6B,SAAS,EAAE,CAAC,CAAC;gBAChF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,gDAAgD;YAChD,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC/C,uDAAuD;gBACvD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC1B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClC,CAAC;gBACD,0BAA0B;gBAC1B,OAAO,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC/E,OAAO,KAAK,CAAC;YACf,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,yBAAyB,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;YACnF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAES,aAAa,CAAC,GAAW;QACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAES,cAAc,CAAC,SAAiB;QACxC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,OAAO,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,+BAA+B;QACnE,CAAC;QACD,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAES,YAAY,CAAC,SAAiB;QACtC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,6CAA6C;YAC7C,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAES,KAAK,CAAC,SAAS;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,EAAE,CAAC;QAEpB,0CAA0C;QAC1C,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,UAAU,EAAE,CAAC;YACvD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,0DAA0D;QAC1D,IAAI,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAClC,OAAO;QACT,CAAC;QAED,wCAAwC;QACxC,MAAM,oBAAoB,GAAG,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC;QACxD,IAAI,oBAAoB,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,oBAAoB,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAI,SAA2B,EAAE,aAAqB,CAAC,EAAE,YAAoB,IAAI;QAC/G,IAAI,SAA4B,CAAC;QAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,OAAO,MAAM,SAAS,EAAE,CAAC;YAC3B,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACb,SAAS,GAAG,KAAc,CAAC;gBAC3B,IAAI,CAAC,KAAK,UAAU,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM;gBACR,CAAC;gBAED,sBAAsB;gBACtB,MAAM,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,MAAM,SAAS,CAAC;IAClB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;CACF"}
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/crawler/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,MAAM,OAAgB,WAAW;IACrB,UAAU,GAAG,KAAK,CAAC;IACZ,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrD,YAAY,GAAG,CAAC,CAAC;IACjB,eAAe,GAAG,CAAC,CAAC;IAIlB,KAAK,CAAC,SAAS;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,EAAE,CAAC;QAEpB,0CAA0C;QAC1C,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,UAAU,EAAE,CAAC;YACvD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,0DAA0D;QAC1D,IAAI,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAClC,OAAO;QACT,CAAC;QAED,wCAAwC;QACxC,MAAM,oBAAoB,GAAG,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC;QACxD,IAAI,oBAAoB,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,oBAAoB,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;CACF"}

package/build/crawler/base.test.js

@@ -2,13 +2,8 @@ import { BaseCrawler } from './base.js';
 /**
  * Concrete implementation of BaseCrawler for testing purposes.
  *
- * Since BaseCrawler is abstract, we create this minimal subclass that:
- * 1. Implements the required abstract `crawl` method
- * 2. Exposes protected BaseCrawler methods via public wrappers
- *
- * The wrapper methods (testShouldCrawl, testMarkUrlAsSeen, etc.) simply call
- * the inherited BaseCrawler methods - we're testing the REAL BaseCrawler
- * implementation, not overridden methods.
+ * Since BaseCrawler is abstract, we create this minimal subclass that
+ * implements the required abstract `crawl` method.
  */
 class TestCrawler extends BaseCrawler {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -21,143 +16,21 @@ class TestCrawler extends BaseCrawler {
             title: 'Test',
         };
     }
-    // Expose protected BaseCrawler methods for testing (these call the real implementations)
-    testShouldCrawl(url) {
-        return this.shouldCrawl(url);
-    }
-    testMarkUrlAsSeen(url) {
-        this.markUrlAsSeen(url);
-    }
-    testGetPathFromUrl(url) {
-        return this.getPathFromUrl(url);
-    }
-    testNormalizeUrl(url) {
-        return this.normalizeUrl(url);
-    }
     async testRateLimit() {
         return this.rateLimit();
     }
-    async testRetryWithBackoff(operation, maxRetries) {
-        return this.retryWithBackoff(operation, maxRetries);
-    }
     testAbort() {
         this.abort();
     }
     get isAbortingFlag() {
         return this.isAborting;
     }
-    testUpdateProgress(description) {
-        this.updateProgress(description);
-    }
-    testAddDiscoveredUrls(count) {
-        this.addDiscoveredUrls(count);
-    }
-    testMarkUrlProcessed(url) {
-        this.markUrlProcessed(url);
-    }
 }
 describe('BaseCrawler', () => {
     let crawler;
     beforeEach(() => {
         crawler = new TestCrawler();
     });
-    describe('constructor', () => {
-        it('should initialize with default values', () => {
-            const defaultCrawler = new TestCrawler();
-            expect(defaultCrawler).toBeDefined();
-        });
-        it('should accept custom maxDepth', () => {
-            const customCrawler = new TestCrawler(10);
-            expect(customCrawler).toBeDefined();
-        });
-        it('should accept custom maxRequestsPerCrawl', () => {
-            const customCrawler = new TestCrawler(4, 500);
-            expect(customCrawler).toBeDefined();
-        });
-        it('should accept progress callback', () => {
-            const progressFn = vi.fn();
-            const customCrawler = new TestCrawler(4, 1000, progressFn);
-            customCrawler.testUpdateProgress('Testing');
-            expect(progressFn).toHaveBeenCalled();
-        });
-    });
-    describe('shouldCrawl', () => {
-        it('should return true for valid URLs', () => {
-            // URLs with .html extension pass the file extension check
-            expect(crawler.testShouldCrawl('https://example.com/docs.html')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/api.html')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/page.htm')).toBe(true);
-        });
-        it('should return false for already seen URLs', () => {
-            crawler.testMarkUrlAsSeen('https://example.com/seen');
-            expect(crawler.testShouldCrawl('https://example.com/seen')).toBe(false);
-        });
-        it('should return false for URLs with hash fragments', () => {
-            expect(crawler.testShouldCrawl('https://example.com/page#section')).toBe(false);
-        });
-        it('should return false for non-HTML files', () => {
-            expect(crawler.testShouldCrawl('https://example.com/image.png')).toBe(false);
-            expect(crawler.testShouldCrawl('https://example.com/doc.pdf')).toBe(false);
-            expect(crawler.testShouldCrawl('https://example.com/script.js')).toBe(false);
-            expect(crawler.testShouldCrawl('https://example.com/style.css')).toBe(false);
-        });
-        it('should return true for HTML files', () => {
-            expect(crawler.testShouldCrawl('https://example.com/page.html')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/page.htm')).toBe(true);
-        });
-        it('should return true for HTML file extensions', () => {
-            expect(crawler.testShouldCrawl('https://example.com/docs.html')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/page.htm')).toBe(true);
-            expect(crawler.testShouldCrawl('https://example.com/index.HTML')).toBe(true);
-        });
-        it('should return false for paths without HTML extension', () => {
-            // The implementation's file extension check treats all paths without
-            // .html/.htm extension as non-HTML files
-            expect(crawler.testShouldCrawl('https://example.com/docs')).toBe(false);
-            expect(crawler.testShouldCrawl('https://example.com/')).toBe(false);
-        });
-        it('should return false for invalid URLs', () => {
-            expect(crawler.testShouldCrawl('not-a-url')).toBe(false);
-            expect(crawler.testShouldCrawl('')).toBe(false);
-        });
-    });
-    describe('markUrlAsSeen', () => {
-        it('should mark URL as seen', () => {
-            const url = 'https://example.com/test.html';
-            expect(crawler.testShouldCrawl(url)).toBe(true);
-            crawler.testMarkUrlAsSeen(url);
-            expect(crawler.testShouldCrawl(url)).toBe(false);
-        });
-    });
-    describe('getPathFromUrl', () => {
-        it('should extract pathname from URL', () => {
-            expect(crawler.testGetPathFromUrl('https://example.com/docs/page')).toBe('/docs/page');
-        });
-        it('should include query params in path', () => {
-            expect(crawler.testGetPathFromUrl('https://example.com/search?q=test')).toBe('/search?q=test');
-        });
-        it('should return original string for invalid URL', () => {
-            expect(crawler.testGetPathFromUrl('not-a-url')).toBe('not-a-url');
-        });
-        it('should handle root path', () => {
-            expect(crawler.testGetPathFromUrl('https://example.com')).toBe('/');
-            expect(crawler.testGetPathFromUrl('https://example.com/')).toBe('/');
-        });
-    });
-    describe('normalizeUrl', () => {
-        it('should remove hash fragments', () => {
-            expect(crawler.testNormalizeUrl('https://example.com/page#section')).toBe('https://example.com/page');
-        });
-        it('should remove trailing slash', () => {
-            expect(crawler.testNormalizeUrl('https://example.com/docs/')).toBe('https://example.com/docs');
-        });
-        it('should preserve query params', () => {
-            expect(crawler.testNormalizeUrl('https://example.com/search?q=test')).toBe('https://example.com/search?q=test');
-        });
-        it('should return original string for invalid URL', () => {
-            expect(crawler.testNormalizeUrl('not-a-url')).toBe('not-a-url');
-        });
-    });
     describe('abort', () => {
         it('should set isAborting flag to true', () => {
             expect(crawler.isAbortingFlag).toBe(false);
@@ -165,61 +38,6 @@ describe('BaseCrawler', () => {
             expect(crawler.isAbortingFlag).toBe(true);
         });
     });
-    describe('retryWithBackoff', () => {
-        it('should return result on successful operation', async () => {
-            const operation = vi.fn().mockResolvedValue('success');
-            const result = await crawler.testRetryWithBackoff(operation);
-            expect(result).toBe('success');
-            expect(operation).toHaveBeenCalledTimes(1);
-        });
-        it('should retry on failure', async () => {
-            const operation = vi.fn().mockRejectedValueOnce(new Error('fail')).mockResolvedValue('success');
-            vi.spyOn(global, 'setTimeout').mockImplementation((callback) => {
-                callback();
-                return 0;
-            });
-            const result = await crawler.testRetryWithBackoff(operation);
-            expect(result).toBe('success');
-            expect(operation).toHaveBeenCalledTimes(2);
-            vi.restoreAllMocks();
-        });
-        it('should throw after max retries', async () => {
-            const operation = vi.fn().mockRejectedValue(new Error('persistent error'));
-            vi.spyOn(global, 'setTimeout').mockImplementation((callback) => {
-                callback();
-                return 0;
-            });
-            await expect(crawler.testRetryWithBackoff(operation, 3)).rejects.toThrow('persistent error');
-            expect(operation).toHaveBeenCalledTimes(3);
-            vi.restoreAllMocks();
-        });
-    });
-    describe('progress tracking', () => {
-        it('should call progress callback with updateProgress', () => {
-            const progressFn = vi.fn();
-            const progressCrawler = new TestCrawler(4, 1000, progressFn);
-            progressCrawler.testUpdateProgress('Processing...');
-            expect(progressFn).toHaveBeenCalledWith(expect.any(Number), 'Processing...');
-        });
-        it('should track discovered URLs', () => {
-            const progressFn = vi.fn();
-            const progressCrawler = new TestCrawler(4, 1000, progressFn);
-            progressCrawler.testAddDiscoveredUrls(5);
-            expect(progressFn).toHaveBeenCalled();
-        });
-        it('should track processed URLs', () => {
-            const progressFn = vi.fn();
-            const progressCrawler = new TestCrawler(4, 1000, progressFn);
-            progressCrawler.testMarkUrlProcessed('https://example.com/page1');
-            expect(progressFn).toHaveBeenCalled();
-        });
-        it('should not throw when progress callback not provided', () => {
-            const noCrawler = new TestCrawler();
-            expect(() => noCrawler.testUpdateProgress('Test')).not.toThrow();
-            expect(() => noCrawler.testAddDiscoveredUrls(5)).not.toThrow();
-            expect(() => noCrawler.testMarkUrlProcessed('https://example.com')).not.toThrow();
-        });
-    });
     describe('crawl', () => {
         it('should yield CrawlResult', async () => {
             const results = [];

package/build/crawler/base.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.test.js","sourceRoot":"","sources":["../../src/crawler/base.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC;;;;;;;;;;GAUG;AACH,MAAM,WAAY,SAAQ,WAAW;IACnC,6DAA6D;IAC7D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAY;QACvB,MAAM;YACJ,GAAG,EAAE,qBAAqB;YAC1B,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,eAAe;YACxB,aAAa,EAAE,MAAM;YACrB,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IAED,yFAAyF;IAClF,eAAe,CAAC,GAAW;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAEM,iBAAiB,CAAC,GAAW;QAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CAAC,GAAW;QACnC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAEM,gBAAgB,CAAC,GAAW;QACjC,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAAI,SAA2B,EAAE,UAAmB;QACnF,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACtD,CAAC;IAEM,SAAS;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,IAAW,cAAc;QACvB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAEM,kBAAkB,CAAC,WAAmB;QAC3C,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAEM,qBAAqB,CAAC,KAAa;QACxC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAEM,oBAAoB,CAAC,GAAW;QACrC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;CACF;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,cAAc,GAAG,IAAI,WAAW,EAAE,CAAC;YACzC,MAAM,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,EAAE,CAAC,CAAC;YAC1C,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9C,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAC3D,aAAa,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,0DAA0D;YAC1D,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,OAAO,CAAC,iBAAiB,CAAC,0BAA0B,CAAC,CAAC;YACtD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,6BAA6B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,gCAAgC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,qEAAqE;YACrE,yCAAyC;YACzC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxE,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,GAAG,GAAG,+BAA+B,CAAC;YAC5C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChD,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,mCAAmC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpE,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,mCAAmC,CAAC,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAClH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3C,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEhG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,QAAoB,EAAE,EAAE;gBACzE,QAAQ,EAAE,CAAC;gBACX,OAAO,CAA8B,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE3C,EAAE,CAAC,eAAe,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;YAE3E,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,QAAoB,EAAE,EAAE;gBACzE,QAAQ,EAAE,CAAC;gBACX,OAAO,CAA8B,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC7F,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE3C,EAAE,CAAC,eAAe,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAE7D,eAAe,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;YAEpD,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAE7D,eAAe,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAEzC,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAE7D,eAAe,CAAC,oBAAoB,CAAC,2BAA2B,CAAC,CAAC;YAElE,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,SAAS,GAAG,IAAI,WAAW,EAAE,CAAC;YAEpC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACjE,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAC/D,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,oBAAoB,CAAC,qBAAqB,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACpF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"base.test.js","sourceRoot":"","sources":["../../src/crawler/base.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC;;;;;GAKG;AACH,MAAM,WAAY,SAAQ,WAAW;IACnC,6DAA6D;IAC7D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAY;QACvB,MAAM;YACJ,GAAG,EAAE,qBAAqB;YAC1B,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,eAAe;YACxB,aAAa,EAAE,MAAM;YACrB,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAEM,SAAS;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,IAAW,cAAc;QACvB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;CACF;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3C,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/crawlee-crawler.js

@@ -500,7 +500,6 @@ export class CrawleeCrawler extends BaseCrawler {
                                 return;
                             }
                             this.queueManager.addResult(result);
-                            this.markUrlProcessed(request.url);
                             break;
                         }
                     }