@cosmocoder/mcp-web-docs 1.0.0

package/build/crawler/content-extractor-types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=content-extractor-types.js.map

package/build/crawler/content-extractor-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-extractor-types.js","sourceRoot":"","sources":["../../src/crawler/content-extractor-types.ts"],"names":[],"mappings":""}

package/build/crawler/content-extractors.d.ts

@@ -0,0 +1,9 @@
+import { StorybookExtractor } from './storybook-extractor.js';
+import { GitHubPagesExtractor } from './github-pages-extractor.js';
+import { DefaultExtractor } from './default-extractor.js';
+export { ContentExtractor, ExtractedContent } from './content-extractor-types.js';
+export declare const contentExtractors: {
+    readonly storybook: StorybookExtractor;
+    readonly github: GitHubPagesExtractor;
+    readonly default: DefaultExtractor;
+};

package/build/crawler/content-extractors.js

@@ -0,0 +1,9 @@
+import { StorybookExtractor } from './storybook-extractor.js';
+import { GitHubPagesExtractor } from './github-pages-extractor.js';
+import { DefaultExtractor } from './default-extractor.js';
+export const contentExtractors = {
+    storybook: new StorybookExtractor(),
+    github: new GitHubPagesExtractor(),
+    default: new DefaultExtractor(),
+};
+//# sourceMappingURL=content-extractors.js.map

package/build/crawler/content-extractors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-extractors.js","sourceRoot":"","sources":["../../src/crawler/content-extractors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAI1D,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,SAAS,EAAE,IAAI,kBAAkB,EAAE;IACnC,MAAM,EAAE,IAAI,oBAAoB,EAAE;IAClC,OAAO,EAAE,IAAI,gBAAgB,EAAE;CACvB,CAAC"}

package/build/crawler/content-utils.d.ts

@@ -0,0 +1,2 @@
+export declare function cleanContent(text: string): string;
+export declare function cleanCodeBlock(code: string): string;

package/build/crawler/content-utils.js

@@ -0,0 +1,22 @@
+// Utility function for cleaning regular content
+export function cleanContent(text) {
+    return text
+        .replace(/\\n/g, '\n') // Convert escaped newlines
+        .replace(/\r\n/g, '\n') // Normalize line endings
+        .replace(/\t/g, '  ') // Convert tabs to spaces
+        .replace(/[^\S\n]+/g, ' ') // Replace multiple spaces with single space (except newlines)
+        .split('\n')
+        .map((line) => line.trimEnd()) // Only trim trailing whitespace, preserve indentation
+        .join('\n')
+        .replace(/\n{3,}/g, '\n\n') // Max 2 consecutive newlines
+        .trim();
+}
+// Utility function specifically for cleaning code blocks
+export function cleanCodeBlock(code) {
+    return code
+        .replace(/^\s+|\s+$/g, '') // Trim whitespace
+        .replace(/\t/g, '  ') // Convert tabs to spaces
+        .replace(/\n{3,}/g, '\n\n') // Reduce multiple blank lines
+        .replace(/\u00A0/g, ' '); // Replace non-breaking spaces
+}
+//# sourceMappingURL=content-utils.js.map

package/build/crawler/content-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-utils.js","sourceRoot":"","sources":["../../src/crawler/content-utils.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,2BAA2B;SACjD,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,yBAAyB;SAChD,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,yBAAyB;SAC9C,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,8DAA8D;SACxF,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,sDAAsD;SACpF,IAAI,CAAC,IAAI,CAAC;SACV,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,6BAA6B;SACxD,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI;SACR,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,kBAAkB;SAC5C,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,yBAAyB;SAC9C,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,8BAA8B;SACzD,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,8BAA8B;AAC5D,CAAC"}

package/build/crawler/content-utils.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/crawler/content-utils.test.js

@@ -0,0 +1,99 @@
+import { cleanContent, cleanCodeBlock } from './content-utils.js';
+describe('Content Utilities', () => {
+    describe('cleanContent', () => {
+        it('should convert escaped newlines to actual newlines', () => {
+            const input = 'Line 1\\nLine 2\\nLine 3';
+            const result = cleanContent(input);
+            expect(result).toBe('Line 1\nLine 2\nLine 3');
+        });
+        it('should normalize Windows line endings', () => {
+            const input = 'Line 1\r\nLine 2\r\nLine 3';
+            const result = cleanContent(input);
+            expect(result).toBe('Line 1\nLine 2\nLine 3');
+        });
+        it('should convert tabs to spaces', () => {
+            const input = 'Line 1\tindented';
+            const result = cleanContent(input);
+            // Tabs are converted to 2 spaces, then multiple spaces become single space
+            expect(result).toBe('Line 1 indented');
+        });
+        it('should replace multiple spaces with single space', () => {
+            const input = 'Too    many     spaces';
+            const result = cleanContent(input);
+            expect(result).toBe('Too many spaces');
+        });
+        it('should trim trailing whitespace from each line', () => {
+            const input = 'Line 1   \nLine 2   ';
+            const result = cleanContent(input);
+            expect(result).toBe('Line 1\nLine 2');
+        });
+        it('should preserve leading indentation', () => {
+            const input = '  indented line\n    more indented';
+            const result = cleanContent(input);
+            expect(result).toBe('indented line\n more indented');
+        });
+        it('should reduce more than 2 consecutive newlines to 2', () => {
+            const input = 'Para 1\n\n\n\n\nPara 2';
+            const result = cleanContent(input);
+            expect(result).toBe('Para 1\n\nPara 2');
+        });
+        it('should trim leading and trailing whitespace', () => {
+            const input = '   \n\nContent here\n\n   ';
+            const result = cleanContent(input);
+            expect(result).toBe('Content here');
+        });
+        it('should handle empty string', () => {
+            expect(cleanContent('')).toBe('');
+        });
+        it('should handle string with only whitespace', () => {
+            expect(cleanContent('   \n\n   ')).toBe('');
+        });
+        it('should handle complex mixed content', () => {
+            const input = '  Title   \r\n\r\n\r\n\r\n  Content with   multiple   spaces  \n\tTabbed';
+            const result = cleanContent(input);
+            // Leading spaces preserved as single space, tabs converted to spaces then collapsed
+            expect(result).toBe('Title\n\n Content with multiple spaces\n Tabbed');
+        });
+    });
+    describe('cleanCodeBlock', () => {
+        it('should trim leading and trailing whitespace', () => {
+            const input = '   \nconst x = 1;\n   ';
+            const result = cleanCodeBlock(input);
+            expect(result).toBe('const x = 1;');
+        });
+        it('should convert tabs to spaces', () => {
+            const input = 'function test() {\n\treturn true;\n}';
+            const result = cleanCodeBlock(input);
+            expect(result).toBe('function test() {\n  return true;\n}');
+        });
+        it('should reduce multiple blank lines', () => {
+            const input = 'line 1\n\n\n\nline 2';
+            const result = cleanCodeBlock(input);
+            expect(result).toBe('line 1\n\nline 2');
+        });
+        it('should replace non-breaking spaces with regular spaces', () => {
+            const input = 'const\u00A0x\u00A0=\u00A01;';
+            const result = cleanCodeBlock(input);
+            expect(result).toBe('const x = 1;');
+        });
+        it('should handle empty string', () => {
+            expect(cleanCodeBlock('')).toBe('');
+        });
+        it('should handle code with multiple formatting issues', () => {
+            const input = '\n\t\tconst x\u00A0= 1;\n\n\n\n\treturn x;\n';
+            const result = cleanCodeBlock(input);
+            expect(result).toBe('const x = 1;\n\n  return x;');
+        });
+        it('should preserve single blank lines', () => {
+            const input = 'line 1\n\nline 2';
+            const result = cleanCodeBlock(input);
+            expect(result).toBe('line 1\n\nline 2');
+        });
+        it('should handle code with mixed indentation', () => {
+            const input = 'function test() {\n\tif (true) {\n\t\treturn;\n\t}\n}';
+            const result = cleanCodeBlock(input);
+            expect(result).toBe('function test() {\n  if (true) {\n    return;\n  }\n}');
+        });
+    });
+});
+//# sourceMappingURL=content-utils.test.js.map

package/build/crawler/content-utils.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-utils.test.js","sourceRoot":"","sources":["../../src/crawler/content-utils.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAElE,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;YAC5D,MAAM,KAAK,GAAG,0BAA0B,CAAC;YACzC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,KAAK,GAAG,4BAA4B,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,KAAK,GAAG,kBAAkB,CAAC;YACjC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,2EAA2E;YAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,KAAK,GAAG,wBAAwB,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,KAAK,GAAG,sBAAsB,CAAC;YACrC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,KAAK,GAAG,oCAAoC,CAAC;YACnD,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,KAAK,GAAG,wBAAwB,CAAC;YACvC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,KAAK,GAAG,4BAA4B,CAAC;YAC3C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,KAAK,GAAG,0EAA0E,CAAC;YACzF,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACnC,oFAAoF;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,KAAK,GAAG,wBAAwB,CAAC;YACvC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,KAAK,GAAG,sCAAsC,CAAC;YACrD,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,KAAK,GAAG,sBAAsB,CAAC;YACrC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;YAChE,MAAM,KAAK,GAAG,6BAA6B,CAAC;YAC5C,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;YAC5D,MAAM,KAAK,GAAG,8CAA8C,CAAC;YAC7D,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,KAAK,GAAG,kBAAkB,CAAC;YACjC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,KAAK,GAAG,uDAAuD,CAAC;YACtE,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/crawlee-crawler.d.ts

@@ -0,0 +1,63 @@
+import { CrawlResult } from '../types.js';
+import { BaseCrawler } from './base.js';
+/** Storage state for authentication (cookies and localStorage) */
+export interface StorageState {
+    cookies: Array<{
+        name: string;
+        value: string;
+        domain: string;
+        path: string;
+        expires?: number;
+        httpOnly?: boolean;
+        secure?: boolean;
+        sameSite?: 'Strict' | 'Lax' | 'None';
+    }>;
+    origins?: Array<{
+        origin: string;
+        localStorage: Array<{
+            name: string;
+            value: string;
+        }>;
+    }>;
+}
+export declare class CrawleeCrawler extends BaseCrawler {
+    private crawler;
+    private queueManager;
+    private storageState?;
+    private isFirstPage;
+    private sessionExpiredError;
+    private expectedUrl;
+    /** The allowed hostname for crawling - pages outside this domain are skipped */
+    private allowedHostname;
+    /** Track pages skipped due to domain mismatch */
+    private skippedExternalPages;
+    /** Optional path prefix to restrict crawling */
+    private pathPrefix?;
+    /**
+     * Set authentication cookies/localStorage to use when crawling
+     */
+    setStorageState(state: StorageState): void;
+    /**
+     * Check if a URL is within the allowed domain for this crawl.
+     * This prevents following redirects or links to external domains.
+     *
+     * @param url - The URL to check
+     * @returns true if the URL is within the allowed domain
+     */
+    private isWithinAllowedDomain;
+    /**
+     * Check if a page appears to be a login/authentication page.
+     * This is used to detect expired sessions during crawling.
+     */
+    private checkForLoginPage;
+    private findContentFrame;
+    private evaluateExtractor;
+    private extractContent;
+    /**
+     * Set an optional path prefix to restrict crawling to URLs under this path.
+     * Must be called before crawl().
+     */
+    setPathPrefix(prefix: string): void;
+    crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
+    abort(): void;
+}

package/build/crawler/crawlee-crawler.js

@@ -0,0 +1,342 @@
+import { PlaywrightCrawler } from 'crawlee';
+import { BaseCrawler } from './base.js';
+import { siteRules } from './site-rules.js';
+import { QueueManager } from './queue-manager.js';
+import { getBrowserConfig } from './browser-config.js';
+import { cleanContent } from './content-utils.js';
+import { logger } from '../util/logger.js';
+import { detectLoginPage, isLoginPageUrl, SessionExpiredError } from '../util/security.js';
+export class CrawleeCrawler extends BaseCrawler {
+    crawler = null;
+    queueManager = new QueueManager();
+    storageState;
+    isFirstPage = true;
+    sessionExpiredError = null;
+    expectedUrl = '';
+    /** The allowed hostname for crawling - pages outside this domain are skipped */
+    allowedHostname = '';
+    /** Track pages skipped due to domain mismatch */
+    skippedExternalPages = 0;
+    /** Optional path prefix to restrict crawling */
+    pathPrefix;
+    /**
+     * Set authentication cookies/localStorage to use when crawling
+     */
+    setStorageState(state) {
+        this.storageState = state;
+        logger.info(`[CrawleeCrawler] Set storage state with ${state.cookies?.length || 0} cookies`);
+    }
+    /**
+     * Check if a URL is within the allowed domain for this crawl.
+     * This prevents following redirects or links to external domains.
+     *
+     * @param url - The URL to check
+     * @returns true if the URL is within the allowed domain
+     */
+    isWithinAllowedDomain(url) {
+        if (!this.allowedHostname) {
+            return true; // No restriction if not set
+        }
+        try {
+            const urlObj = new URL(url);
+            const hostname = urlObj.hostname.toLowerCase();
+            const allowed = this.allowedHostname.toLowerCase();
+            // Exact match
+            if (hostname === allowed) {
+                return true;
+            }
+            // Allow subdomains (e.g., docs.example.com when allowed is example.com)
+            // But NOT the other way around (github.com is not allowed for *.github.io)
+            if (hostname.endsWith('.' + allowed)) {
+                return true;
+            }
+            return false;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Check if a page appears to be a login/authentication page.
+     * This is used to detect expired sessions during crawling.
+     */
+    async checkForLoginPage(page, currentUrl) {
+        // Only check the first page with high scrutiny
+        // (subsequent pages being login pages might be intentional navigation)
+        if (!this.isFirstPage) {
+            return false;
+        }
+        // Check URL pattern first (fast)
+        if (isLoginPageUrl(currentUrl)) {
+            logger.warn(`[CrawleeCrawler] First page URL matches login pattern: ${currentUrl}`);
+            return true;
+        }
+        // Check page content
+        try {
+            const bodyText = await page.evaluate(() => document.body?.textContent || '');
+            const pageHtml = await page.content();
+            const detection = detectLoginPage(bodyText + pageHtml, currentUrl);
+            if (detection.isLoginPage && detection.confidence >= 0.5) {
+                logger.warn(`[CrawleeCrawler] First page appears to be a login page (confidence: ${detection.confidence.toFixed(2)})`);
+                logger.debug(`[CrawleeCrawler] Detection reasons: ${detection.reasons.join(', ')}`);
+                // Store the error for throwing later (can't throw from request handler)
+                this.sessionExpiredError = new SessionExpiredError(`Authentication session has expired - crawled page is a login page`, this.expectedUrl, currentUrl, detection);
+                return true;
+            }
+        }
+        catch (error) {
+            logger.debug(`[CrawleeCrawler] Error checking for login page:`, error);
+        }
+        return false;
+    }
+    async findContentFrame(page) {
+        const frames = await page.frames();
+        const contentFrames = await Promise.all(frames.map(async (frame) => {
+            try {
+                const hasContent = await frame
+                    .evaluate(() => {
+                    return document.querySelector('.sbdocs-content, #docs-root, .docs-story, [class*="story-"]') !== null;
+                })
+                    .catch(() => false);
+                if (hasContent) {
+                    await Promise.all([
+                        frame.waitForLoadState('domcontentloaded'),
+                        frame
+                            .waitForLoadState('networkidle', { timeout: 5000 })
+                            .catch(() => logger.debug('Frame network idle timeout - continuing anyway')),
+                    ]);
+                    return frame;
+                }
+            }
+            catch (error) {
+                logger.debug('Error checking frame', { error: String(error) });
+            }
+            return null;
+        }));
+        const frame = contentFrames.find((f) => f !== null) || null;
+        if (frame) {
+            logger.debug('Found content in iframe');
+            await new Promise((resolve) => setTimeout(resolve, 2000));
+        }
+        return frame;
+    }
+    async evaluateExtractor(context, extractor) {
+        const extractorCode = extractor.constructor.toString();
+        return context.evaluate(async (code) => {
+            const ExtractorClass = new Function(`return ${code}`)();
+            const extractor = new ExtractorClass();
+            const result = await extractor.extractContent(document);
+            return result.content;
+        }, extractorCode);
+    }
+    async extractContent(page, siteType, extractor) {
+        let content = '';
+        let extractorUsed = extractor.constructor.name;
+        try {
+            if (siteType === 'storybook') {
+                // Try iframe first
+                const frame = await this.findContentFrame(page);
+                if (frame) {
+                    content = await this.evaluateExtractor(frame, extractor);
+                }
+                // Fallback to main page
+                if (!content) {
+                    content = await this.evaluateExtractor(page, extractor);
+                }
+            }
+            else {
+                content = await this.evaluateExtractor(page, extractor);
+            }
+        }
+        catch {
+            content = await page.evaluate(() => document.body.textContent || '');
+            extractorUsed = 'ErrorFallback';
+        }
+        return { content, extractorUsed };
+    }
+    /**
+     * Set an optional path prefix to restrict crawling to URLs under this path.
+     * Must be called before crawl().
+     */
+    setPathPrefix(prefix) {
+        this.pathPrefix = prefix;
+        logger.info(`[CrawleeCrawler] Path prefix restriction set: ${prefix}`);
+    }
+    async *crawl(url) {
+        logger.debug(`[${this.constructor.name}] Starting crawl of: ${url}`);
+        // Reset state for this crawl
+        this.isFirstPage = true;
+        this.sessionExpiredError = null;
+        this.expectedUrl = url;
+        this.skippedExternalPages = 0;
+        // Extract and store the allowed hostname from the initial URL
+        try {
+            this.allowedHostname = new URL(url).hostname;
+            logger.info(`[CrawleeCrawler] Domain restriction: only crawling pages on ${this.allowedHostname}`);
+        }
+        catch {
+            this.allowedHostname = '';
+        }
+        await this.queueManager.initialize(url, this.pathPrefix);
+        // Build crawler options with optional authentication
+        const crawlerOptions = getBrowserConfig(this.queueManager.getRequestQueue() ?? undefined);
+        // If we have storage state (auth cookies), configure the browser to use them
+        if (this.storageState) {
+            logger.info(`[CrawleeCrawler] Using authenticated session with ${this.storageState.cookies?.length || 0} cookies`);
+            crawlerOptions.launchContext = {
+                ...crawlerOptions.launchContext,
+                launchOptions: {
+                    ...crawlerOptions.launchContext?.launchOptions,
+                },
+            };
+            crawlerOptions.browserPoolOptions = {
+                ...crawlerOptions.browserPoolOptions,
+                preLaunchHooks: [
+                    async (pageId) => {
+                        // Storage state will be set in preNavigationHooks instead
+                        logger.debug(`[CrawleeCrawler] Browser launching for page ${pageId}`);
+                    },
+                ],
+            };
+            // Add cookies via preNavigationHooks
+            const existingHooks = crawlerOptions.preNavigationHooks || [];
+            crawlerOptions.preNavigationHooks = [
+                ...existingHooks,
+                async ({ page }) => {
+                    if (this.storageState?.cookies) {
+                        logger.debug(`[CrawleeCrawler] Setting ${this.storageState.cookies.length} cookies before navigation`);
+                        await page.context().addCookies(this.storageState.cookies);
+                    }
+                },
+            ];
+        }
+        this.crawler = new PlaywrightCrawler({
+            ...crawlerOptions,
+            requestHandler: async ({ request, page, enqueueLinks, log }) => {
+                if (this.isAborting) {
+                    log.debug('Crawl aborted');
+                    return;
+                }
+                try {
+                    // Wait for initial page load
+                    await Promise.all([
+                        page.waitForLoadState('domcontentloaded'),
+                        page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => log.debug('Network idle timeout - continuing anyway')),
+                    ]);
+                    // Get the actual URL after any redirects
+                    const actualUrl = page.url();
+                    // Check if the page redirected outside the allowed domain
+                    if (!this.isWithinAllowedDomain(actualUrl)) {
+                        const requestedHostname = new URL(request.url).hostname;
+                        const actualHostname = new URL(actualUrl).hostname;
+                        if (this.isFirstPage) {
+                            // First page redirected outside domain - likely auth redirect (session expired)
+                            logger.warn(`[CrawleeCrawler] First page redirected outside allowed domain: ${requestedHostname} → ${actualHostname}`);
+                            if (this.storageState) {
+                                // We had auth but got redirected - session expired
+                                this.sessionExpiredError = new SessionExpiredError(`Authentication session has expired - page redirected to external domain (${actualHostname})`, this.expectedUrl, actualUrl, { isLoginPage: true, confidence: 1.0, reasons: [`Redirected from ${requestedHostname} to ${actualHostname}`] });
+                                log.error(`Session expired - redirected to external domain: ${actualHostname}. Aborting crawl.`);
+                                this.abort();
+                                return;
+                            }
+                            else {
+                                // No auth but redirected - might be site misconfiguration
+                                log.error(`First page redirected to external domain: ${actualHostname}. Aborting crawl.`);
+                                this.abort();
+                                return;
+                            }
+                        }
+                        else {
+                            // Subsequent page redirected outside domain - skip it
+                            this.skippedExternalPages++;
+                            log.warning(`Skipping page that redirected outside domain: ${request.url} → ${actualUrl} (skipped ${this.skippedExternalPages} external pages)`);
+                            return;
+                        }
+                    }
+                    // Check for login page on first page (detects expired sessions)
+                    if (this.isFirstPage && this.storageState) {
+                        const isLoginPage = await this.checkForLoginPage(page, actualUrl);
+                        if (isLoginPage) {
+                            log.error('Session appears expired - first page is a login page. Aborting crawl.');
+                            this.abort();
+                            return;
+                        }
+                        this.isFirstPage = false;
+                    }
+                    else if (this.isFirstPage) {
+                        this.isFirstPage = false;
+                    }
+                    // Detect site type and get extractor
+                    for (const rule of siteRules) {
+                        if (await rule.detect(page)) {
+                            if (rule.prepare) {
+                                await rule.prepare(page, log);
+                            }
+                            await this.queueManager.handleQueueAndLinks(enqueueLinks, log, rule);
+                            const title = await page.title();
+                            const { content, extractorUsed } = await this.extractContent(page, rule.type, rule.extractor);
+                            const result = {
+                                url: request.url,
+                                path: new URL(request.url).pathname + new URL(request.url).search,
+                                content: cleanContent(content),
+                                title,
+                                extractorUsed,
+                            };
+                            this.queueManager.addResult(result);
+                            this.markUrlProcessed(request.url);
+                            break;
+                        }
+                    }
+                }
+                catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    log.error(`Error processing ${request.url}: ${errorMessage}`);
+                }
+            },
+        });
+        try {
+            const crawlerPromise = this.crawler.run();
+            while (!this.isAborting) {
+                if (this.queueManager.hasEnoughResults()) {
+                    for (const result of await this.queueManager.processBatch()) {
+                        yield result;
+                    }
+                }
+                if (await Promise.race([crawlerPromise.then(() => true), new Promise((resolve) => setTimeout(() => resolve(false), 100))]))
+                    break;
+            }
+            await crawlerPromise;
+            logger.debug('Crawler finished');
+            // Log summary of domain-restricted crawling
+            if (this.skippedExternalPages > 0) {
+                logger.warn(`[CrawleeCrawler] Skipped ${this.skippedExternalPages} pages that redirected outside the allowed domain (${this.allowedHostname})`);
+            }
+            // Check if we detected an expired session during crawling
+            if (this.sessionExpiredError) {
+                throw this.sessionExpiredError;
+            }
+            for (const result of await this.queueManager.processBatch()) {
+                yield result;
+            }
+        }
+        catch (error) {
+            // Re-throw session expired errors as-is
+            if (error instanceof SessionExpiredError) {
+                throw error;
+            }
+            logger.debug('Crawler error:', error);
+            throw error;
+        }
+        finally {
+            await this.queueManager.cleanup();
+            this.crawler = null;
+        }
+    }
+    abort() {
+        super.abort();
+        if (this.crawler) {
+            this.crawler.teardown().catch((err) => logger.error('Failed to teardown crawler:', err));
+        }
+    }
+}
+//# sourceMappingURL=crawlee-crawler.js.map

package/build/crawler/crawlee-crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawlee-crawler.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAoB3F,MAAM,OAAO,cAAe,SAAQ,WAAW;IACrC,OAAO,GAA6B,IAAI,CAAC;IACzC,YAAY,GAAiB,IAAI,YAAY,EAAE,CAAC;IAChD,YAAY,CAAgB;IAC5B,WAAW,GAAY,IAAI,CAAC;IAC5B,mBAAmB,GAA+B,IAAI,CAAC;IACvD,WAAW,GAAW,EAAE,CAAC;IACjC,gFAAgF;IACxE,eAAe,GAAW,EAAE,CAAC;IACrC,iDAAiD;IACzC,oBAAoB,GAAW,CAAC,CAAC;IACzC,gDAAgD;IACxC,UAAU,CAAU;IAE5B;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,2CAA2C,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/F,CAAC;IAED;;;;;;OAMG;IACK,qBAAqB,CAAC,GAAW;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,CAAC,4BAA4B;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEnD,cAAc;YACd,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAAC,IAAU,EAAE,UAAkB;QAC5D,+CAA+C;QAC/C,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,iCAAiC;QACjC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,0DAA0D,UAAU,EAAE,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,GAAG,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEnE,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBACzD,MAAM,CAAC,IAAI,CAAC,uEAAuE,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvH,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEpF,wEAAwE;gBACxE,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,mEAAmE,EACnE,IAAI,CAAC,WAAW,EAChB,UAAU,EACV,SAAS,CACV,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAU;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,KAAK;qBAC3B,QAAQ,CAAC,GAAG,EAAE;oBACb,OAAO,QAAQ,CAAC,aAAa,CAAC,6DAA6D,CAAC,KAAK,IAAI,CAAC;gBACxG,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;gBAEtB,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,KAAK,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBAC1C,KAAK;6BACF,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;6BAClD,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;qBAC/E,CAAC,CAAC;oBACH,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;QAC5D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,OAAqB,EAAE,SAA2B;QAChF,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QACvD,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAC7C,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACxD,OAAO,MAAM,CAAC,OAAO,CAAC;QACxB,CAAC,EAAE,aAAa,CAAC,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,IAAU,EACV,QAAgB,EAChB,SAA2B;QAE3B,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC;QAE/C,IAAI,CAAC;YACH,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC3D,CAAC;gBAED,wBAAwB;gBACxB,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,aAAa,GAAG,eAAe,CAAC;QAClC,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,iDAAiD,MAAM,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,wBAAwB,GAAG,EAAE,CAAC,CAAC;QAErE,6BAA6B;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC;QACvB,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAE9B,8DAA8D;QAC9D,IAAI,CAAC;YACH,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,+DAA+D,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QACrG,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzD,qDAAqD;QACrD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,SAAS,CAAC,CAAC;QAE1F,6EAA6E;QAC7E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;YACnH,cAAc,CAAC,aAAa,GAAG;gBAC7B,GAAG,cAAc,CAAC,aAAa;gBAC/B,aAAa,EAAE;oBACb,GAAG,cAAc,CAAC,aAAa,EAAE,aAAa;iBAC/C;aACF,CAAC;YACF,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,cAAc,CAAC,kBAAkB;gBACpC,cAAc,EAAE;oBACd,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,0DAA0D;wBAC1D,MAAM,CAAC,KAAK,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;oBACxE,CAAC;iBACF;aACF,CAAC;YACF,qCAAqC;YACrC,MAAM,aAAa,GAAG,cAAc,CAAC,kBAAkB,IAAI,EAAE,CAAC;YAC9D,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,aAAa;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;oBACjB,IAAI,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,4BAA4B,CAAC,CAAC;wBACvG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC;YACnC,GAAG,cAAc;YACjB,cAAc,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE;gBAC7D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;oBAC3B,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC;oBACH,6BAA6B;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;qBAC3H,CAAC,CAAC;oBAEH,yCAAyC;oBACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAE7B,0DAA0D;oBAC1D,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;wBACxD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC;wBAEnD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,gFAAgF;4BAChF,MAAM,CAAC,IAAI,CAAC,kEAAkE,iBAAiB,MAAM,cAAc,EAAE,CAAC,CAAC;4BAEvH,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gCACtB,mDAAmD;gCACnD,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,4EAA4E,cAAc,GAAG,EAC7F,IAAI,CAAC,WAAW,EAChB,SAAS,EACT,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,mBAAmB,iBAAiB,OAAO,cAAc,EAAE,CAAC,EAAE,CAC/G,CAAC;gCACF,GAAG,CAAC,KAAK,CAAC,oDAAoD,cAAc,mBAAmB,CAAC,CAAC;gCACjG,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;iCAAM,CAAC;gCACN,0DAA0D;gCAC1D,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,mBAAmB,CAAC,CAAC;gCAC1F,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,sDAAsD;4BACtD,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC5B,GAAG,CAAC,OAAO,CACT,iDAAiD,OAAO,CAAC,GAAG,MAAM,SAAS,aAAa,IAAI,CAAC,oBAAoB,kBAAkB,CACpI,CAAC;4BACF,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;wBAClE,IAAI,WAAW,EAAE,CAAC;4BAChB,GAAG,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;4BACnF,IAAI,CAAC,KAAK,EAAE,CAAC;4BACb,OAAO;wBACT,CAAC;wBACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;yBAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBAC5B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;oBAED,qCAAqC;oBACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;4BAC5B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gCACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAChC,CAAC;4BAED,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;4BAErE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACjC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;4BAE9F,MAAM,MAAM,GAAgB;gCAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;gCAChB,IAAI,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;gCACjE,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC;gCAC9B,KAAK;gCACL,aAAa;6BACd,CAAC;4BAEF,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BACpC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;4BACnC,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5E,GAAG,CAAC,KAAK,CAAC,oBAAoB,OAAO,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YAE1C,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,EAAE,CAAC;oBACzC,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;wBAC5D,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;oBAAE,MAAM;YACpI,CAAC;YAED,MAAM,cAAc,CAAC;YACrB,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAEjC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CACT,4BAA4B,IAAI,CAAC,oBAAoB,sDAAsD,IAAI,CAAC,eAAe,GAAG,CACnI,CAAC;YACJ,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,CAAC;YACf,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wCAAwC;YACxC,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}

package/build/crawler/crawlee-crawler.test.d.ts

@@ -0,0 +1 @@
+export {};