@cosmocoder/mcp-web-docs 1.0.0

package/build/crawler/crawlee-crawler.test.js

@@ -0,0 +1,280 @@
+const mockQueueManager = {
+    initialize: vi.fn().mockResolvedValue(undefined),
+    getRequestQueue: vi.fn().mockReturnValue({}),
+    handleQueueAndLinks: vi.fn().mockResolvedValue(undefined),
+    addResult: vi.fn(),
+    hasEnoughResults: vi.fn().mockReturnValue(false),
+    processBatch: vi.fn().mockResolvedValue([]),
+    cleanup: vi.fn().mockResolvedValue(undefined),
+};
+vi.mock('./queue-manager.js', () => ({
+    QueueManager: function () {
+        return mockQueueManager;
+    },
+}));
+vi.mock('./browser-config.js', () => ({
+    getBrowserConfig: vi.fn().mockReturnValue({
+        requestQueue: {},
+        preNavigationHooks: [],
+        launchContext: {},
+        browserPoolOptions: {},
+    }),
+}));
+vi.mock('./site-rules.js', () => ({
+    siteRules: [
+        {
+            type: 'default',
+            extractor: {
+                extractContent: vi.fn().mockResolvedValue({ content: 'Extracted content', metadata: {} }),
+            },
+            detect: vi.fn().mockResolvedValue(true),
+        },
+    ],
+}));
+// Mock PlaywrightCrawler
+const mockCrawlerRun = vi.fn().mockResolvedValue(undefined);
+const mockCrawlerTeardown = vi.fn().mockResolvedValue(undefined);
+vi.mock('crawlee', () => ({
+    PlaywrightCrawler: function (options) {
+        // Store the request handler for testing
+        global.__requestHandler = options.requestHandler;
+        return {
+            run: mockCrawlerRun,
+            teardown: mockCrawlerTeardown,
+        };
+    },
+}));
+// Import after mocking
+import { CrawleeCrawler } from './crawlee-crawler.js';
+describe('CrawleeCrawler', () => {
+    let crawler;
+    beforeEach(() => {
+        vi.clearAllMocks();
+        crawler = new CrawleeCrawler();
+        mockQueueManager.hasEnoughResults.mockReturnValue(false);
+        mockQueueManager.processBatch.mockResolvedValue([]);
+    });
+    describe('constructor', () => {
+        it('should initialize with default values', () => {
+            expect(crawler).toBeDefined();
+        });
+        it('should accept custom maxDepth and maxRequestsPerCrawl', () => {
+            const customCrawler = new CrawleeCrawler(10, 500);
+            expect(customCrawler).toBeDefined();
+        });
+        it('should accept progress callback', () => {
+            const progressFn = vi.fn();
+            const progressCrawler = new CrawleeCrawler(4, 1000, progressFn);
+            expect(progressCrawler).toBeDefined();
+        });
+    });
+    describe('setStorageState', () => {
+        it('should accept storage state', () => {
+            const state = {
+                cookies: [{ name: 'session', value: 'abc123', domain: 'example.com', path: '/' }],
+            };
+            crawler.setStorageState(state);
+            // No error means success
+            expect(true).toBe(true);
+        });
+        it('should accept storage state with origins', () => {
+            const state = {
+                cookies: [{ name: 'session', value: 'abc123', domain: 'example.com', path: '/' }],
+                origins: [
+                    {
+                        origin: 'https://example.com',
+                        localStorage: [{ name: 'token', value: 'xyz' }],
+                    },
+                ],
+            };
+            crawler.setStorageState(state);
+            expect(true).toBe(true);
+        });
+    });
+    describe('crawl', () => {
+        it('should initialize queue manager with URL', async () => {
+            // Set up processBatch to return results immediately to end the crawl
+            mockCrawlerRun.mockResolvedValueOnce(undefined);
+            mockQueueManager.processBatch.mockResolvedValueOnce([]);
+            const results = [];
+            for await (const result of crawler.crawl('https://example.com/docs')) {
+                results.push(result);
+            }
+            expect(mockQueueManager.initialize).toHaveBeenCalledWith('https://example.com/docs', undefined);
+        });
+        it('should yield results from queue manager', async () => {
+            const mockResults = [
+                { url: 'https://example.com/page1', path: '/page1', content: 'Page 1', title: 'Page 1' },
+                { url: 'https://example.com/page2', path: '/page2', content: 'Page 2', title: 'Page 2' },
+            ];
+            // Since hasEnoughResults returns false, processBatch is only called once
+            // at the end of crawl (line 388 in crawlee-crawler.ts), so we only need one mock value
+            mockQueueManager.processBatch.mockResolvedValueOnce(mockResults);
+            const results = [];
+            for await (const result of crawler.crawl('https://example.com')) {
+                results.push(result);
+            }
+            expect(results).toEqual(mockResults);
+        });
+        it('should cleanup queue manager after crawl', async () => {
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for await (const _ of crawler.crawl('https://example.com')) {
+                // Just consume results
+            }
+            expect(mockQueueManager.cleanup).toHaveBeenCalled();
+        });
+        it('should process batch when enough results accumulated', async () => {
+            const mockResults = [{ url: 'https://example.com/page1', path: '/page1', content: 'Page 1', title: 'Page 1' }];
+            mockQueueManager.hasEnoughResults.mockReturnValueOnce(true).mockReturnValue(false);
+            mockQueueManager.processBatch.mockResolvedValueOnce(mockResults).mockResolvedValueOnce([]);
+            const results = [];
+            for await (const result of crawler.crawl('https://example.com')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(1);
+        });
+        it('should set allowed hostname from URL', async () => {
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for await (const _ of crawler.crawl('https://docs.example.com/guide')) {
+                // Just consume results
+            }
+            // Verify through the initialize call
+            expect(mockQueueManager.initialize).toHaveBeenCalledWith('https://docs.example.com/guide', undefined);
+        });
+    });
+    describe('abort', () => {
+        it('should stop the crawler', async () => {
+            // Make run() hang until we resolve it, so abort() can be called while crawler exists
+            let resolveRun;
+            const runPromise = new Promise((resolve) => {
+                resolveRun = resolve;
+            });
+            mockCrawlerRun.mockReturnValueOnce(runPromise);
+            // Create a crawler that we can abort
+            const abortableCrawler = new CrawleeCrawler();
+            // Start consuming the generator - this creates the crawler
+            const generator = abortableCrawler.crawl('https://example.com');
+            // Get the first value to start the generator (this creates the crawler)
+            const firstResultPromise = generator.next();
+            // Give the generator time to start and create the crawler
+            await new Promise((resolve) => setTimeout(resolve, 10));
+            // Now abort - the crawler exists at this point
+            abortableCrawler.abort();
+            // Let the run() complete so the generator can finish
+            resolveRun();
+            // Wait for the generator to complete
+            await firstResultPromise;
+            const results = [];
+            for await (const result of generator) {
+                results.push(result);
+            }
+            expect(mockCrawlerTeardown).toHaveBeenCalled();
+        });
+    });
+    describe('domain restriction', () => {
+        it('should extract hostname from URL for domain restriction', async () => {
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for await (const _ of crawler.crawl('https://subdomain.example.com/path')) {
+                // Just consume results
+            }
+            expect(mockQueueManager.initialize).toHaveBeenCalledWith('https://subdomain.example.com/path', undefined);
+        });
+        it('should pass path prefix to queue manager when set', async () => {
+            crawler.setPathPrefix('/docs/api');
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for await (const _ of crawler.crawl('https://example.com/docs/api')) {
+                // Just consume results
+            }
+            expect(mockQueueManager.initialize).toHaveBeenCalledWith('https://example.com/docs/api', '/docs/api');
+        });
+    });
+    describe('authentication', () => {
+        it('should configure crawler with storage state when set', async () => {
+            const state = {
+                cookies: [{ name: 'auth', value: 'token123', domain: 'example.com', path: '/' }],
+            };
+            crawler.setStorageState(state);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for await (const _ of crawler.crawl('https://example.com')) {
+                // Just consume results
+            }
+            // Verify queue manager was initialized (auth is handled internally)
+            expect(mockQueueManager.initialize).toHaveBeenCalled();
+        });
+    });
+    describe('isWithinAllowedDomain', () => {
+        // Access the private method through the class prototype for testing
+        it('should handle URL parsing for domain check', async () => {
+            // This is tested indirectly through the crawl method
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for await (const _ of crawler.crawl('https://example.com')) {
+                // Just consume results
+            }
+            // No errors means domain parsing worked
+            expect(mockQueueManager.initialize).toHaveBeenCalled();
+        });
+    });
+    describe('error handling', () => {
+        it('should cleanup on error', async () => {
+            mockCrawlerRun.mockRejectedValueOnce(new Error('Crawl failed'));
+            await expect(async () => {
+                // eslint-disable-next-line @typescript-eslint/no-unused-vars
+                for await (const _ of crawler.crawl('https://example.com')) {
+                    // Just consume results
+                }
+            }).rejects.toThrow('Crawl failed');
+            expect(mockQueueManager.cleanup).toHaveBeenCalled();
+        });
+        it('should handle invalid URLs gracefully', async () => {
+            // The crawler should handle this internally
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for await (const _ of crawler.crawl('https://example.com')) {
+                // Just consume results
+            }
+            expect(mockQueueManager.initialize).toHaveBeenCalled();
+        });
+    });
+});
+describe('StorageState interface', () => {
+    it('should allow cookies with all optional properties', () => {
+        const state = {
+            cookies: [
+                {
+                    name: 'session',
+                    value: 'abc',
+                    domain: 'example.com',
+                    path: '/',
+                    expires: 1234567890,
+                    httpOnly: true,
+                    secure: true,
+                    sameSite: 'Strict',
+                },
+            ],
+        };
+        expect(state.cookies).toHaveLength(1);
+        expect(state.cookies[0].sameSite).toBe('Strict');
+    });
+    it('should allow minimal cookie definition', () => {
+        const state = {
+            cookies: [{ name: 'token', value: 'xyz', domain: 'test.com', path: '/' }],
+        };
+        expect(state.cookies[0].expires).toBeUndefined();
+    });
+    it('should allow origins for localStorage', () => {
+        const state = {
+            cookies: [],
+            origins: [
+                {
+                    origin: 'https://example.com',
+                    localStorage: [
+                        { name: 'key1', value: 'value1' },
+                        { name: 'key2', value: 'value2' },
+                    ],
+                },
+            ],
+        };
+        expect(state.origins).toHaveLength(1);
+        expect(state.origins[0].localStorage).toHaveLength(2);
+    });
+});
+//# sourceMappingURL=crawlee-crawler.test.js.map

package/build/crawler/crawlee-crawler.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawlee-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.test.ts"],"names":[],"mappings":"AAEA,MAAM,gBAAgB,GAAG;IACvB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IAChD,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;IAC5C,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IACzD,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;IAChD,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;IAC3C,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;CAC9C,CAAC;AAEF,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,YAAY,EAAE;QACZ,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC;IACpC,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;QACxC,YAAY,EAAE,EAAE;QAChB,kBAAkB,EAAE,EAAE;QACtB,aAAa,EAAE,EAAE;QACjB,kBAAkB,EAAE,EAAE;KACvB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;IAChC,SAAS,EAAE;QACT;YACE,IAAI,EAAE,SAAS;YACf,SAAS,EAAE;gBACT,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;aAC1F;YACD,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC;KACF;CACF,CAAC,CAAC,CAAC;AAEJ,yBAAyB;AACzB,MAAM,cAAc,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAC5D,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEjE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,iBAAiB,EAAE,UAAU,OAAqC;QAChE,wCAAwC;QACvC,MAAyC,CAAC,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC;QACrF,OAAO;YACL,GAAG,EAAE,cAAc;YACnB,QAAQ,EAAE,mBAAmB;SAC9B,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,uBAAuB;AACvB,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AAEpE,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAuB,CAAC;IAE5B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QAC/B,gBAAgB,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACzD,gBAAgB,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,aAAa,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YAClD,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAChE,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aAClF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,yBAAyB;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;gBACjF,OAAO,EAAE;oBACP;wBACE,MAAM,EAAE,qBAAqB;wBAC7B,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;qBAChD;iBACF;aACF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,qEAAqE;YACrE,cAAc,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;YAChD,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAExD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QAClG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAkB;gBACjC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;gBACxF,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;aACzF,CAAC;YAEF,yEAAyE;YACzE,uFAAuF;YACvF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;YAEjE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,WAAW,GAAkB,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE9H,gBAAgB,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACnF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3F,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC;gBACtE,uBAAuB;YACzB,CAAC;YAED,qCAAqC;YACrC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,gCAAgC,EAAE,SAAS,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,qFAAqF;YACrF,IAAI,UAAsB,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAC/C,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAE/C,qCAAqC;YACrC,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;YAE9C,2DAA2D;YAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAEhE,wEAAwE;YACxE,MAAM,kBAAkB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5C,0DAA0D;YAC1D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAExD,+CAA+C;YAC/C,gBAAgB,CAAC,KAAK,EAAE,CAAC;YAEzB,qDAAqD;YACrD,UAAW,EAAE,CAAC;YAEd,qCAAqC;YACrC,MAAM,kBAAkB,CAAC;YACzB,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC1E,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,oCAAoC,EAAE,SAAS,CAAC,CAAC;QAC5G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YAEnC,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;gBACpE,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aACjF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,oEAAoE;YACpE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,oEAAoE;QACpE,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,qDAAqD;YACrD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,wCAAwC;YACxC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,cAAc,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAEhE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;gBACtB,6DAA6D;gBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBAC3D,uBAAuB;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEnC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,4CAA4C;YAC5C,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,KAAK;oBACZ,MAAM,EAAE,aAAa;oBACrB,IAAI,EAAE,GAAG;oBACT,OAAO,EAAE,UAAU;oBACnB,QAAQ,EAAE,IAAI;oBACd,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,QAAQ;iBACnB;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;SAC1E,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,EAAE;YACX,OAAO,EAAE;gBACP;oBACE,MAAM,EAAE,qBAAqB;oBAC7B,YAAY,EAAE;wBACZ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;wBACjC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;qBAClC;iBACF;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/default-extractor.d.ts

@@ -0,0 +1,4 @@
+import { ContentExtractor, ExtractedContent } from './content-extractor-types.js';
+export declare class DefaultExtractor implements ContentExtractor {
+    extractContent(document: Document): Promise<ExtractedContent>;
+}

package/build/crawler/default-extractor.js

@@ -0,0 +1,26 @@
+export class DefaultExtractor {
+    async extractContent(document) {
+        // Remove common non-content elements
+        document.querySelectorAll('style, script, nav, header, footer').forEach((el) => el.remove());
+        // Get main content
+        const main = document.querySelector('main, article, [role="main"]');
+        const contentElement = main ? main.cloneNode(true) : document.body;
+        // Extract title and description
+        const title = contentElement.querySelector('h1')?.textContent?.trim();
+        const firstParagraph = contentElement.querySelector('h1 + p')?.textContent?.trim();
+        return {
+            content: contentElement.textContent?.trim() || '',
+            metadata: {
+                type: 'overview',
+                pattern: {
+                    name: title || '',
+                    type: 'component',
+                    description: firstParagraph || '',
+                    usageContexts: [],
+                    relatedPatterns: [],
+                },
+            },
+        };
+    }
+}
+//# sourceMappingURL=default-extractor.js.map

package/build/crawler/default-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-extractor.js","sourceRoot":"","sources":["../../src/crawler/default-extractor.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,gBAAgB;IAC3B,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,qCAAqC;QACrC,QAAQ,CAAC,gBAAgB,CAAC,oCAAoC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7F,mBAAmB;QACnB,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,8BAA8B,CAAC,CAAC;QACpE,MAAM,cAAc,GAAG,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;QAEhF,gCAAgC;QAChC,MAAM,KAAK,GAAG,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QACtE,MAAM,cAAc,GAAG,cAAc,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAEnF,OAAO;YACL,OAAO,EAAE,cAAc,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE;YACjD,QAAQ,EAAE;gBACR,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE;oBACP,IAAI,EAAE,KAAK,IAAI,EAAE;oBACjB,IAAI,EAAE,WAAW;oBACjB,WAAW,EAAE,cAAc,IAAI,EAAE;oBACjC,aAAa,EAAE,EAAE;oBACjB,eAAe,EAAE,EAAE;iBACpB;aACF;SACF,CAAC;IACJ,CAAC;CACF"}

package/build/crawler/default-extractor.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/crawler/default-extractor.test.js

@@ -0,0 +1,200 @@
+import { DefaultExtractor } from './default-extractor.js';
+import { JSDOM } from 'jsdom';
+describe('DefaultExtractor', () => {
+    let extractor;
+    beforeEach(() => {
+        extractor = new DefaultExtractor();
+    });
+    function createDocument(html) {
+        const dom = new JSDOM(html);
+        return dom.window.document;
+    }
+    describe('extractContent', () => {
+        it('should extract content from main element', async () => {
+            const html = `
+        <html>
+          <body>
+            <nav>Navigation</nav>
+            <main>
+              <h1>Main Title</h1>
+              <p>Main content here</p>
+            </main>
+            <footer>Footer</footer>
+          </body>
+        </html>
+      `;
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.content).toContain('Main Title');
+            expect(result.content).toContain('Main content here');
+            expect(result.content).not.toContain('Navigation');
+            expect(result.content).not.toContain('Footer');
+        });
+        it('should extract content from article element', async () => {
+            const html = `
+        <html>
+          <body>
+            <header>Header</header>
+            <article>
+              <h1>Article Title</h1>
+              <p>Article content</p>
+            </article>
+          </body>
+        </html>
+      `;
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.content).toContain('Article Title');
+            expect(result.content).toContain('Article content');
+            expect(result.content).not.toContain('Header');
+        });
+        it('should extract content from role="main" element', async () => {
+            const html = `
+        <html>
+          <body>
+            <nav>Nav</nav>
+            <div role="main">
+              <h1>Role Main Title</h1>
+              <p>Role main content</p>
+            </div>
+          </body>
+        </html>
+      `;
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.content).toContain('Role Main Title');
+            expect(result.content).toContain('Role main content');
+            expect(result.content).not.toContain('Nav');
+        });
+        it('should fall back to body when no main content element', async () => {
+            const html = `
+        <html>
+          <body>
+            <div>
+              <h1>Page Title</h1>
+              <p>Page content</p>
+            </div>
+          </body>
+        </html>
+      `;
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.content).toContain('Page Title');
+            expect(result.content).toContain('Page content');
+        });
+        it('should remove script and style elements', async () => {
+            const html = `
+        <html>
+          <body>
+            <style>.hidden { display: none; }</style>
+            <script>console.log('secret');</script>
+            <main>
+              <h1>Visible Content</h1>
+            </main>
+          </body>
+        </html>
+      `;
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.content).toContain('Visible Content');
+            expect(result.content).not.toContain('hidden');
+            expect(result.content).not.toContain('secret');
+        });
+        it('should extract title from h1', async () => {
+            const html = `
+        <html>
+          <body>
+            <main>
+              <h1>Component Name</h1>
+              <p>Description paragraph</p>
+            </main>
+          </body>
+        </html>
+      `;
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.metadata.pattern?.name).toBe('Component Name');
+        });
+        it('should extract description from first paragraph after h1', async () => {
+            const html = `
+        <html>
+          <body>
+            <main>
+              <h1>Component Name</h1>
+              <p>This is the component description.</p>
+              <p>This is additional content.</p>
+            </main>
+          </body>
+        </html>
+      `;
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.metadata.pattern?.description).toBe('This is the component description.');
+        });
+        it('should return overview type metadata', async () => {
+            const html = '<html><body><main><h1>Test</h1></main></body></html>';
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.metadata.type).toBe('overview');
+        });
+        it('should return component type pattern', async () => {
+            const html = '<html><body><main><h1>Test</h1></main></body></html>';
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.metadata.pattern?.type).toBe('component');
+        });
+        it('should handle empty document', async () => {
+            const html = '<html><body></body></html>';
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.content).toBe('');
+            expect(result.metadata.type).toBe('overview');
+        });
+        it('should handle document with only whitespace', async () => {
+            const html = '<html><body>   \n\n   </body></html>';
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.content).toBe('');
+        });
+        it('should initialize usageContexts and relatedPatterns as empty arrays', async () => {
+            const html = '<html><body><main><h1>Test</h1></main></body></html>';
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.metadata.pattern?.usageContexts).toEqual([]);
+            expect(result.metadata.pattern?.relatedPatterns).toEqual([]);
+        });
+        it('should handle missing h1', async () => {
+            const html = `
+        <html>
+          <body>
+            <main>
+              <h2>Subheading</h2>
+              <p>Content here</p>
+            </main>
+          </body>
+        </html>
+      `;
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.metadata.pattern?.name).toBe('');
+            expect(result.content).toContain('Content here');
+        });
+        it('should handle missing description paragraph', async () => {
+            const html = `
+        <html>
+          <body>
+            <main>
+              <h1>Title Only</h1>
+              <div>Some div content</div>
+            </main>
+          </body>
+        </html>
+      `;
+            const doc = createDocument(html);
+            const result = await extractor.extractContent(doc);
+            expect(result.metadata.pattern?.name).toBe('Title Only');
+            expect(result.metadata.pattern?.description).toBe('');
+        });
+    });
+});
+//# sourceMappingURL=default-extractor.test.js.map

package/build/crawler/default-extractor.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-extractor.test.js","sourceRoot":"","sources":["../../src/crawler/default-extractor.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,IAAI,SAA2B,CAAC;IAEhC,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,SAAS,cAAc,CAAC,IAAY;QAClC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,OAAO,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;IAC7B,CAAC;IAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,IAAI,GAAG;;;;;;;;;;;OAWZ,CAAC;YACF,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,IAAI,GAAG;;;;;;;;;;OAUZ,CAAC;YACF,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,IAAI,GAAG;;;;;;;;;;OAUZ,CAAC;YACF,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,IAAI,GAAG;;;;;;;;;OASZ,CAAC;YACF,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,IAAI,GAAG;;;;;;;;;;OAUZ,CAAC;YACF,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,IAAI,GAAG;;;;;;;;;OASZ,CAAC;YACF,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACxE,MAAM,IAAI,GAAG;;;;;;;;;;OAUZ,CAAC;YACF,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QAC1F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,IAAI,GAAG,sDAAsD,CAAC;YACpE,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,IAAI,GAAG,sDAAsD,CAAC;YACpE,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,IAAI,GAAG,4BAA4B,CAAC;YAC1C,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAChC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,IAAI,GAAG,sCAAsC,CAAC;YACpD,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;YACnF,MAAM,IAAI,GAAG,sDAAsD,CAAC;YACpE,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC3D,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,IAAI,GAAG;;;;;;;;;OASZ,CAAC;YACF,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,IAAI,GAAG;;;;;;;;;OASZ,CAAC;YACF,MAAM,GAAG,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAEnD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/default.d.ts

@@ -0,0 +1,11 @@
+import { CrawlResult } from '../types.js';
+import { BaseCrawler } from './base.js';
+export declare class DefaultCrawler extends BaseCrawler {
+    private readonly BATCH_SIZE;
+    private readonly FETCH_TIMEOUT;
+    constructor(maxDepth?: number, maxRequestsPerCrawl?: number, onProgress?: (progress: number, description: string) => void);
+    crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
+    private processPageWithRetry;
+    private extractLinks;
+    private extractTitle;
+}