@cosmocoder/mcp-web-docs 1.0.0

package/build/crawler/github.test.js

@@ -0,0 +1,326 @@
+import { GitHubCrawler } from './github.js';
+describe('GitHubCrawler', () => {
+    let crawler;
+    beforeEach(() => {
+        fetchMock.resetMocks();
+        crawler = new GitHubCrawler();
+        // Mock rateLimit to skip delays - the method is in BaseCrawler
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        vi.spyOn(crawler, 'rateLimit').mockResolvedValue(undefined);
+    });
+    describe('constructor', () => {
+        it('should initialize with default values', () => {
+            const defaultCrawler = new GitHubCrawler();
+            expect(defaultCrawler).toBeDefined();
+        });
+        it('should accept custom maxDepth and maxRequestsPerCrawl', () => {
+            const customCrawler = new GitHubCrawler(10, 500);
+            expect(customCrawler).toBeDefined();
+        });
+        it('should accept GitHub token', () => {
+            const tokenCrawler = new GitHubCrawler(4, 1000, 'github_token_123');
+            expect(tokenCrawler).toBeDefined();
+        });
+        it('should accept progress callback', () => {
+            const progressFn = vi.fn();
+            const progressCrawler = new GitHubCrawler(4, 1000, undefined, progressFn);
+            expect(progressCrawler).toBeDefined();
+        });
+    });
+    describe('crawl', () => {
+        it('should reject invalid GitHub URLs', async () => {
+            const results = [];
+            // Non-GitHub URL
+            for await (const result of crawler.crawl('https://example.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(0);
+        });
+        it('should reject URLs without owner/repo', async () => {
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(0);
+        });
+        it('should crawl documentation directory when found', async () => {
+            // First call: list root directory
+            fetchMock.mockResponseOnce(JSON.stringify([
+                { path: 'docs', type: 'dir', name: 'docs', url: 'https://api.github.com/repos/owner/repo/contents/docs' },
+                { path: 'src', type: 'dir', name: 'src', url: 'https://api.github.com/repos/owner/repo/contents/src' },
+            ]));
+            // Second call: list docs directory
+            fetchMock.mockResponseOnce(JSON.stringify([
+                {
+                    path: 'docs/guide.md',
+                    type: 'file',
+                    name: 'guide.md',
+                    url: 'https://api.github.com/repos/owner/repo/contents/docs/guide.md',
+                },
+                { path: 'docs/api.md', type: 'file', name: 'api.md', url: 'https://api.github.com/repos/owner/repo/contents/docs/api.md' },
+            ]));
+            // Third call: fetch guide.md content
+            fetchMock.mockResponseOnce('# Guide\n\nThis is the guide content.');
+            // Fourth call: fetch api.md content
+            fetchMock.mockResponseOnce('# API Reference\n\nAPI documentation.');
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(2);
+            expect(results[0].path).toBe('docs/guide.md');
+            expect(results[0].content).toContain('Guide');
+            expect(results[1].path).toBe('docs/api.md');
+        });
+        it('should handle .git extension in repo URL', async () => {
+            const rootFiles = [
+                { path: 'README.md', type: 'file', name: 'README.md', url: 'https://api.github.com/repos/owner/repo/contents/README.md' },
+            ];
+            // First call: findDocumentationDirs checks root
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // Second call: processDirectory fetches root again (no doc dirs found)
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // Third call: fetch README.md content
+            fetchMock.mockResponseOnce('# README\n\nProject readme.');
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo.git')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(1);
+            expect(results[0].title).toBe('README');
+        });
+        it('should skip non-markdown files', async () => {
+            const rootFiles = [
+                { path: 'index.js', type: 'file', name: 'index.js', url: 'https://api.github.com/repos/owner/repo/contents/index.js' },
+                { path: 'style.css', type: 'file', name: 'style.css', url: 'https://api.github.com/repos/owner/repo/contents/style.css' },
+                { path: 'README.md', type: 'file', name: 'README.md', url: 'https://api.github.com/repos/owner/repo/contents/README.md' },
+            ];
+            // First call: findDocumentationDirs checks root
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // Second call: processDirectory fetches root again (no doc dirs found)
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // Third call: fetch README.md content
+            fetchMock.mockResponseOnce('# README');
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(1);
+            expect(results[0].path).toBe('README.md');
+        });
+        it('should handle various markdown extensions', async () => {
+            const rootFiles = [
+                { path: 'doc.md', type: 'file', name: 'doc.md', url: 'https://api.github.com/repos/owner/repo/contents/doc.md' },
+                { path: 'page.mdx', type: 'file', name: 'page.mdx', url: 'https://api.github.com/repos/owner/repo/contents/page.mdx' },
+                {
+                    path: 'guide.markdown',
+                    type: 'file',
+                    name: 'guide.markdown',
+                    url: 'https://api.github.com/repos/owner/repo/contents/guide.markdown',
+                },
+            ];
+            // First call: findDocumentationDirs checks root
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // Second call: processDirectory fetches root again (no doc dirs found)
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // File content fetches
+            fetchMock.mockResponseOnce('# Doc');
+            fetchMock.mockResponseOnce('# Page');
+            fetchMock.mockResponseOnce('# Guide');
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(3);
+        });
+        it('should skip directories like node_modules, vendor, test, etc.', async () => {
+            fetchMock.mockResponseOnce(JSON.stringify([
+                {
+                    path: 'node_modules',
+                    type: 'dir',
+                    name: 'node_modules',
+                    url: 'https://api.github.com/repos/owner/repo/contents/node_modules',
+                },
+                { path: 'vendor', type: 'dir', name: 'vendor', url: 'https://api.github.com/repos/owner/repo/contents/vendor' },
+                { path: 'test', type: 'dir', name: 'test', url: 'https://api.github.com/repos/owner/repo/contents/test' },
+                { path: 'docs', type: 'dir', name: 'docs', url: 'https://api.github.com/repos/owner/repo/contents/docs' },
+            ]));
+            fetchMock.mockResponseOnce(JSON.stringify([
+                {
+                    path: 'docs/guide.md',
+                    type: 'file',
+                    name: 'guide.md',
+                    url: 'https://api.github.com/repos/owner/repo/contents/docs/guide.md',
+                },
+            ]));
+            fetchMock.mockResponseOnce('# Guide');
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            // Should only get the docs/guide.md, not files from skipped directories
+            expect(results).toHaveLength(1);
+            expect(results[0].path).toBe('docs/guide.md');
+        });
+        it('should handle GitHub API rate limit error', async () => {
+            fetchMock.mockResponseOnce('', { status: 403 });
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            // Should return empty due to rate limit
+            expect(results).toHaveLength(0);
+        });
+        it('should handle API errors gracefully', async () => {
+            fetchMock.mockResponseOnce('', { status: 404 });
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(0);
+        });
+        it('should use GitHub token in headers when provided', async () => {
+            const tokenCrawler = new GitHubCrawler(4, 1000, 'test_token_123');
+            // Mock rateLimit for new crawler instance
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            vi.spyOn(tokenCrawler, 'rateLimit').mockResolvedValue(undefined);
+            fetchMock.mockResponseOnce(JSON.stringify([]));
+            const results = [];
+            for await (const result of tokenCrawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(fetchMock).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
+                headers: expect.objectContaining({
+                    Authorization: 'token test_token_123',
+                }),
+            }));
+        });
+        it('should extract title from file path', async () => {
+            const rootFiles = [
+                {
+                    path: 'getting-started.md',
+                    type: 'file',
+                    name: 'getting-started.md',
+                    url: 'https://api.github.com/repos/owner/repo/contents/getting-started.md',
+                },
+                {
+                    path: 'api_reference.md',
+                    type: 'file',
+                    name: 'api_reference.md',
+                    url: 'https://api.github.com/repos/owner/repo/contents/api_reference.md',
+                },
+            ];
+            // First call: findDocumentationDirs checks root
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // Second call: processDirectory fetches root again (no doc dirs found)
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // File content fetches
+            fetchMock.mockResponseOnce('# Content');
+            fetchMock.mockResponseOnce('# Content');
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(results[0].title).toBe('Getting Started');
+            expect(results[1].title).toBe('Api Reference');
+        });
+        it('should construct correct GitHub blob URLs', async () => {
+            // First call: findDocumentationDirs checks root - find docs directory
+            fetchMock.mockResponseOnce(JSON.stringify([{ path: 'docs', type: 'dir', name: 'docs', url: 'https://api.github.com/repos/owner/repo/contents/docs' }]));
+            // Second call: processDirectory fetches docs directory contents
+            fetchMock.mockResponseOnce(JSON.stringify([
+                {
+                    path: 'docs/guide.md',
+                    type: 'file',
+                    name: 'guide.md',
+                    url: 'https://api.github.com/repos/owner/repo/contents/docs/guide.md',
+                },
+            ]));
+            // Third call: fetch file content
+            fetchMock.mockResponseOnce('# Guide');
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(results[0].url).toBe('https://github.com/owner/repo/blob/main/docs/guide.md');
+        });
+        it('should handle fetch errors for file content', async () => {
+            fetchMock.mockResponseOnce(JSON.stringify([
+                { path: 'guide.md', type: 'file', name: 'guide.md', url: 'https://api.github.com/repos/owner/repo/contents/guide.md' },
+            ]));
+            fetchMock.mockRejectOnce(new Error('Network error'));
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            // Should skip files that fail to fetch
+            expect(results).toHaveLength(0);
+        });
+        it('should validate GitHub API response structure', async () => {
+            // Mock invalid response structure
+            fetchMock.mockResponseOnce(JSON.stringify({ invalid: 'structure' }));
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(0);
+        });
+        it('should find multiple documentation directories', async () => {
+            fetchMock.mockResponseOnce(JSON.stringify([
+                { path: 'docs', type: 'dir', name: 'docs', url: 'https://api.github.com/repos/owner/repo/contents/docs' },
+                { path: 'guide', type: 'dir', name: 'guide', url: 'https://api.github.com/repos/owner/repo/contents/guide' },
+            ]));
+            fetchMock.mockResponseOnce(JSON.stringify([
+                { path: 'docs/api.md', type: 'file', name: 'api.md', url: 'https://api.github.com/repos/owner/repo/contents/docs/api.md' },
+            ]));
+            fetchMock.mockResponseOnce('# API');
+            fetchMock.mockResponseOnce(JSON.stringify([
+                {
+                    path: 'guide/intro.md',
+                    type: 'file',
+                    name: 'intro.md',
+                    url: 'https://api.github.com/repos/owner/repo/contents/guide/intro.md',
+                },
+            ]));
+            fetchMock.mockResponseOnce('# Intro');
+            const results = [];
+            for await (const result of crawler.crawl('https://github.com/owner/repo')) {
+                results.push(result);
+            }
+            expect(results).toHaveLength(2);
+        });
+        it('should stop crawling when aborted', async () => {
+            const rootFiles = [
+                { path: 'doc1.md', type: 'file', name: 'doc1.md', url: 'https://api.github.com/repos/owner/repo/contents/doc1.md' },
+                { path: 'doc2.md', type: 'file', name: 'doc2.md', url: 'https://api.github.com/repos/owner/repo/contents/doc2.md' },
+            ];
+            // First call: findDocumentationDirs checks root
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // Second call: processDirectory fetches root again (no doc dirs found)
+            fetchMock.mockResponseOnce(JSON.stringify(rootFiles));
+            // Third call: fetch doc1.md content
+            fetchMock.mockResponseOnce('# Doc 1');
+            const results = [];
+            const abortableCrawler = new GitHubCrawler();
+            // Mock rateLimit for new crawler instance
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            vi.spyOn(abortableCrawler, 'rateLimit').mockResolvedValue(undefined);
+            // Get the generator
+            const generator = abortableCrawler.crawl('https://github.com/owner/repo');
+            // Get first result
+            const first = await generator.next();
+            if (!first.done) {
+                results.push(first.value);
+            }
+            // Abort before getting second result
+            abortableCrawler.isAborting = true;
+            // Try to get more results
+            for await (const result of generator) {
+                results.push(result);
+            }
+            // Should only have the first result
+            expect(results).toHaveLength(1);
+        });
+    });
+});
+//# sourceMappingURL=github.test.js.map

package/build/crawler/github.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.test.js","sourceRoot":"","sources":["../../src/crawler/github.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG5C,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,IAAI,OAAsB,CAAC;IAE3B,UAAU,CAAC,GAAG,EAAE;QACd,SAAS,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO,GAAG,IAAI,aAAa,EAAE,CAAC;QAC9B,+DAA+D;QAC/D,8DAA8D;QAC9D,EAAE,CAAC,KAAK,CAAC,OAAc,EAAE,WAAW,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,cAAc,GAAG,IAAI,aAAa,EAAE,CAAC;YAC3C,MAAM,CAAC,cAAc,CAAC,CAAC,WAAW,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACjD,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,YAAY,GAAG,IAAI,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,kBAAkB,CAAC,CAAC;YACpE,MAAM,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;YAC1E,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAElC,iBAAiB;YACjB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC;gBAC3E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAElC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBAC/D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,kCAAkC;YAClC,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;gBACzG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,sDAAsD,EAAE;aACvG,CAAC,CACH,CAAC;YACF,mCAAmC;YACnC,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,eAAe;oBACrB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,gEAAgE;iBACtE;gBACD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,8DAA8D,EAAE;aAC3H,CAAC,CACH,CAAC;YACF,qCAAqC;YACrC,SAAS,CAAC,gBAAgB,CAAC,uCAAuC,CAAC,CAAC;YACpE,oCAAoC;YACpC,SAAS,CAAC,gBAAgB,CAAC,uCAAuC,CAAC,CAAC;YAEpE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC9C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAC9C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,SAAS,GAAG;gBAChB,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,4DAA4D,EAAE;aAC1H,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,sCAAsC;YACtC,SAAS,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CAAC;YAE1D,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,EAAE,CAAC;gBAC9E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,SAAS,GAAG;gBAChB,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,2DAA2D,EAAE;gBACtH,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,4DAA4D,EAAE;gBACzH,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,4DAA4D,EAAE;aAC1H,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,sCAAsC;YACtC,SAAS,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YAEvC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG;gBAChB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,yDAAyD,EAAE;gBAChH,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,2DAA2D,EAAE;gBACtH;oBACE,IAAI,EAAE,gBAAgB;oBACtB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,gBAAgB;oBACtB,GAAG,EAAE,iEAAiE;iBACvE;aACF,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uBAAuB;YACvB,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACpC,SAAS,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YACrC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,cAAc;oBACpB,IAAI,EAAE,KAAK;oBACX,IAAI,EAAE,cAAc;oBACpB,GAAG,EAAE,+DAA+D;iBACrE;gBACD,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,yDAAyD,EAAE;gBAC/G,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;gBACzG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;aAC1G,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,eAAe;oBACrB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,gEAAgE;iBACtE;aACF,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,wEAAwE;YACxE,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,SAAS,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAEhD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,wCAAwC;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,SAAS,CAAC,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAEhD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,GAAG,IAAI,aAAa,CAAC,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAClE,0CAA0C;YAC1C,8DAA8D;YAC9D,EAAE,CAAC,KAAK,CAAC,YAAmB,EAAE,WAAW,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAExE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;YAE/C,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC/E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAClB,MAAM,CAAC,gBAAgB,CAAC;gBACtB,OAAO,EAAE,MAAM,CAAC,gBAAgB,CAAC;oBAC/B,aAAa,EAAE,sBAAsB;iBACtC,CAAC;aACH,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,SAAS,GAAG;gBAChB;oBACE,IAAI,EAAE,oBAAoB;oBAC1B,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,oBAAoB;oBAC1B,GAAG,EAAE,qEAAqE;iBAC3E;gBACD;oBACE,IAAI,EAAE,kBAAkB;oBACxB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,kBAAkB;oBACxB,GAAG,EAAE,mEAAmE;iBACzE;aACF,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uBAAuB;YACvB,SAAS,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACxC,SAAS,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YAExC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACjD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,sEAAsE;YACtE,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE,CAAC,CAAC,CAC5H,CAAC;YACF,gEAAgE;YAChE,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,eAAe;oBACrB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,gEAAgE;iBACtE;aACF,CAAC,CACH,CAAC;YACF,iCAAiC;YACjC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;QACvF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,2DAA2D,EAAE;aACvH,CAAC,CACH,CAAC;YACF,SAAS,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAErD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,uCAAuC;YACvC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,kCAAkC;YAClC,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;YAErE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,uDAAuD,EAAE;gBACzG,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,wDAAwD,EAAE;aAC7G,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,8DAA8D,EAAE;aAC3H,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACpC,SAAS,CAAC,gBAAgB,CACxB,IAAI,CAAC,SAAS,CAAC;gBACb;oBACE,IAAI,EAAE,gBAAgB;oBACtB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,iEAAiE;iBACvE;aACF,CAAC,CACH,CAAC;YACF,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC;gBAC1E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,SAAS,GAAG;gBAChB,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,0DAA0D,EAAE;gBACnH,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,0DAA0D,EAAE;aACpH,CAAC;YACF,gDAAgD;YAChD,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,uEAAuE;YACvE,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YACtD,oCAAoC;YACpC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,MAAM,gBAAgB,GAAG,IAAI,aAAa,EAAE,CAAC;YAC7C,0CAA0C;YAC1C,8DAA8D;YAC9D,EAAE,CAAC,KAAK,CAAC,gBAAuB,EAAE,WAAW,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAE5E,oBAAoB;YACpB,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAE1E,mBAAmB;YACnB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5B,CAAC;YAED,qCAAqC;YACpC,gBAAuD,CAAC,UAAU,GAAG,IAAI,CAAC;YAE3E,0BAA0B;YAC1B,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,oCAAoC;YACpC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/puppeteer.d.ts

@@ -0,0 +1,16 @@
+import { CrawlResult } from '../types.js';
+import { BaseCrawler } from './base.js';
+export declare class PuppeteerCrawler extends BaseCrawler {
+    private browser?;
+    private readonly userAgent;
+    private readonly LINK_GROUP_SIZE;
+    private curCrawlCount;
+    crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
+    private setupPage;
+    private crawlSitePages;
+    private gotoPageAndHandleRedirects;
+    private processPage;
+    private getLinksFromPage;
+    private groupLinks;
+    abort(): void;
+}

package/build/crawler/puppeteer.js

@@ -0,0 +1,191 @@
+import puppeteer from 'puppeteer';
+import { BaseCrawler } from './base.js';
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+export class PuppeteerCrawler extends BaseCrawler {
+    browser;
+    userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36';
+    LINK_GROUP_SIZE = 2;
+    curCrawlCount = 0;
+    async *crawl(url) {
+        try {
+            this.browser = await puppeteer.launch({
+                headless: true,
+                args: [
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-dev-shm-usage',
+                    '--disable-gpu',
+                    '--window-size=1280,800'
+                ]
+            });
+            const page = await this.browser.newPage();
+            await this.setupPage(page);
+            const visitedUrls = new Set();
+            yield* this.crawlSitePages(page, new URL(url), 0, visitedUrls);
+        }
+        finally {
+            await this.browser?.close();
+        }
+    }
+    async setupPage(page) {
+        await page.setUserAgent(this.userAgent);
+        await page.setViewport({ width: 1280, height: 800 });
+        // Block only unnecessary resources
+        await page.setRequestInterception(true);
+        page.on('request', request => {
+            const resourceType = request.resourceType();
+            if (['image', 'media', 'font'].includes(resourceType)) {
+                request.abort();
+            }
+            else {
+                request.continue();
+            }
+        });
+        // Handle JavaScript errors
+        page.on('pageerror', error => {
+            console.warn('Page error:', error);
+        });
+        // Handle console messages
+        page.on('console', (msg) => {
+            const type = msg.type();
+            if (type === 'error' || type === 'warn') {
+                console.debug(`Console ${type}:`, msg.text());
+            }
+        });
+    }
+    async *crawlSitePages(page, curUrl, depth, visitedUrls) {
+        const urlStr = curUrl.toString();
+        if (visitedUrls.has(urlStr) || !this.shouldCrawl(urlStr) || depth > this.maxDepth) {
+            return;
+        }
+        try {
+            // Rate limiting
+            await this.rateLimit();
+            // Navigate to page with proper redirect handling
+            await this.gotoPageAndHandleRedirects(page, urlStr);
+            // Extract content
+            const { content, title, links } = await this.processPage(page, curUrl);
+            visitedUrls.add(urlStr);
+            this.markUrlAsSeen(urlStr);
+            this.curCrawlCount++;
+            yield {
+                url: urlStr,
+                path: this.getPathFromUrl(urlStr),
+                content,
+                title
+            };
+            // Process links in batches
+            if (depth < this.maxDepth && this.curCrawlCount < this.maxRequestsPerCrawl) {
+                const linkGroups = this.groupLinks(links);
+                for (const linkGroup of linkGroups) {
+                    for (const link of linkGroup) {
+                        if (this.curCrawlCount >= this.maxRequestsPerCrawl) {
+                            return;
+                        }
+                        yield* this.crawlSitePages(page, new URL(link), depth + 1, visitedUrls);
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.error(`Error crawling ${urlStr}:`, error);
+        }
+    }
+    async gotoPageAndHandleRedirects(page, url) {
+        const MAX_PAGE_WAIT_MS = 5000;
+        await page.goto(url, {
+            timeout: 0,
+            waitUntil: 'networkidle2'
+        });
+        let responseEventOccurred = false;
+        const responseHandler = () => responseEventOccurred = true;
+        const responseWatcher = new Promise((resolve) => {
+            setTimeout(() => {
+                if (!responseEventOccurred) {
+                    resolve();
+                }
+                else {
+                    setTimeout(() => resolve(), MAX_PAGE_WAIT_MS);
+                }
+            }, 500);
+        });
+        page.on('response', responseHandler);
+        await Promise.race([responseWatcher, page.waitForNavigation()]);
+        page.off('response', responseHandler);
+    }
+    async processPage(page, url) {
+        // Wait for dynamic content
+        try {
+            await page.waitForFunction(() => {
+                const mainContent = document.querySelector('main') || document.querySelector('.content') || document.querySelector('#content');
+                return mainContent && mainContent.children.length > 0;
+            }, { timeout: 5000 });
+        }
+        catch (error) {
+            console.warn('Timeout waiting for main content, proceeding anyway');
+        }
+        // Extract content using Readability
+        const html = await page.content();
+        const dom = new JSDOM(html, { url: url.toString() });
+        const reader = new Readability(dom.window.document, {
+            charThreshold: 20,
+            nbTopCandidates: 5,
+            maxElemsToParse: 10000
+        });
+        const article = reader.parse();
+        if (!article) {
+            throw new Error('Failed to parse page content');
+        }
+        // Extract links
+        const links = await this.getLinksFromPage(page, url);
+        return {
+            content: article.textContent,
+            title: article.title,
+            links
+        };
+    }
+    async getLinksFromPage(page, curUrl) {
+        const links = await page.$$eval('a', (links) => links.map((a) => a.href));
+        const cleanedLinks = links
+            .map(link => {
+            try {
+                const url = new URL(link);
+                url.hash = ''; // Remove hash
+                return url.href;
+            }
+            catch {
+                return null;
+            }
+        })
+            .filter((link) => {
+            if (!link)
+                return false;
+            try {
+                const url = new URL(link);
+                return (url.pathname.startsWith(curUrl.pathname) &&
+                    url.hostname === curUrl.hostname &&
+                    link !== curUrl.href);
+            }
+            catch {
+                return false;
+            }
+        });
+        return Array.from(new Set(cleanedLinks));
+    }
+    groupLinks(links) {
+        return links.reduce((acc, link, i) => {
+            const groupIndex = Math.floor(i / this.LINK_GROUP_SIZE);
+            if (!acc[groupIndex]) {
+                acc.push([]);
+            }
+            acc[groupIndex].push(link);
+            return acc;
+        }, []);
+    }
+    abort() {
+        super.abort();
+        void this.browser?.close();
+    }
+}
+//# sourceMappingURL=puppeteer.js.map

package/build/crawler/puppeteer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer.js","sourceRoot":"","sources":["../../src/crawler/puppeteer.ts"],"names":[],"mappings":"AAAA,OAAO,SAA4C,MAAM,WAAW,CAAC;AAErE,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,OAAO,gBAAiB,SAAQ,WAAW;IACvC,OAAO,CAAW;IACT,SAAS,GAAG,qHAAqH,CAAC;IAClI,eAAe,GAAG,CAAC,CAAC;IAC7B,aAAa,GAAG,CAAC,CAAC;IAE1B,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC;gBACpC,QAAQ,EAAE,IAAI;gBACd,IAAI,EAAE;oBACJ,cAAc;oBACd,0BAA0B;oBAC1B,yBAAyB;oBACzB,eAAe;oBACf,wBAAwB;iBACzB;aACF,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC1C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE3B,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;YACtC,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;QACjE,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;QAC9B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,IAAU;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAErD,mCAAmC;QACnC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE;YAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YAC5C,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACtD,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,2BAA2B;QAC3B,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE;YAC3B,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,0BAA0B;QAC1B,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAmB,EAAE,EAAE;YACzC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxC,OAAO,CAAC,KAAK,CAAC,WAAW,IAAI,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,CAAC,cAAc,CAC3B,IAAU,EACV,MAAW,EACX,KAAa,EACb,WAAwB;QAExB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QAEjC,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClF,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAEvB,iDAAiD;YACjD,MAAM,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEpD,kBAAkB;YAClB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEvE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACxB,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC3B,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,MAAM;gBACJ,GAAG,EAAE,MAAM;gBACX,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;gBACjC,OAAO;gBACP,KAAK;aACN,CAAC;YAEF,2BAA2B;YAC3B,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC3E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC1C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;oBACnC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;4BACnD,OAAO;wBACT,CAAC;wBACD,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;oBAC1E,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,MAAM,GAAG,EAAE,KAAK,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,0BAA0B,CAAC,IAAU,EAAE,GAAW;QAC9D,MAAM,gBAAgB,GAAG,IAAI,CAAC;QAE9B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACnB,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,cAAc;SAC1B,CAAC,CAAC;QAEH,IAAI,qBAAqB,GAAG,KAAK,CAAC;QAClC,MAAM,eAAe,GAAG,GAAG,EAAE,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAE3D,MAAM,eAAe,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACpD,UAAU,CAAC,GAAG,EAAE;gBACd,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBAC3B,OAAO,EAAE,CAAC;gBACZ,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,gBAAgB,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC,EAAE,GAAG,CAAC,CAAC;QACV,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QACrC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,eAAe,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;QAChE,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;IACxC,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAU,EAAE,GAAQ;QAC5C,2BAA2B;QAC3B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE;gBAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;gBAC/H,OAAO,WAAW,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACxD,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACtE,CAAC;QAED,oCAAoC;QACpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE;YAClD,aAAa,EAAE,EAAE;YACjB,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,KAAK;SACvB,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QAED,gBAAgB;QAChB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAErD,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,WAAW;YAC5B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,KAAK;SACN,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAU,EAAE,MAAW;QACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAE1E,MAAM,YAAY,GAAG,KAAK;aACvB,GAAG,CAAC,IAAI,CAAC,EAAE;YACV,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC1B,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,cAAc;gBAC7B,OAAO,GAAG,CAAC,IAAI,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE;YAC/B,IAAI,CAAC,IAAI;gBAAE,OAAO,KAAK,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC1B,OAAO,CACL,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC;oBACxC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ;oBAChC,IAAI,KAAK,MAAM,CAAC,IAAI,CACrB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;QAEL,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;IAC3C,CAAC;IAEO,UAAU,CAAC,KAAe;QAChC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;YACxD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACrB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,CAAC;YACD,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAgB,CAAC,CAAC;IACvB,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;IAC7B,CAAC;CACF"}

package/build/crawler/queue-manager.d.ts

@@ -0,0 +1,43 @@
+import { RequestQueue, Log, EnqueueLinksOptions } from 'crawlee';
+import { CrawlResult } from '../types.js';
+import { SiteDetectionRule } from './site-rules.js';
+export declare class QueueManager {
+    private requestQueue;
+    private websiteId;
+    private results;
+    private static readonly BATCH_SIZE;
+    /** Optional path prefix to restrict crawling to URLs under this path */
+    private pathPrefix;
+    /** The allowed hostname - only URLs with this exact hostname (or its subdomains) are allowed */
+    private allowedHostname;
+    /** Count of URLs filtered due to path prefix mismatch */
+    private filteredByPathCount;
+    /** Count of URLs filtered due to hostname mismatch */
+    private filteredByHostnameCount;
+    initialize(url: string, pathPrefix?: string): Promise<void>;
+    getFilteredByPathCount(): number;
+    getFilteredByHostnameCount(): number;
+    /**
+     * Check if a hostname matches the allowed hostname.
+     * Allows exact match or subdomains of the allowed hostname.
+     * Does NOT allow sibling subdomains or parent domains.
+     *
+     * @example
+     * If allowedHostname is 'docs.example.com':
+     * - 'docs.example.com' → true (exact match)
+     * - 'api.docs.example.com' → true (subdomain)
+     * - 'example.com' → false (parent domain)
+     * - 'python.example.com' → false (sibling subdomain)
+     */
+    private isHostnameAllowed;
+    handleQueueAndLinks(enqueueLinks: (options: EnqueueLinksOptions) => Promise<{
+        processedRequests: {
+            uniqueKey: string;
+        }[];
+    }>, log: Log, rule: SiteDetectionRule): Promise<void>;
+    processBatch(): Promise<CrawlResult[]>;
+    addResult(result: CrawlResult): void;
+    hasEnoughResults(): boolean;
+    getRequestQueue(): RequestQueue | null;
+    cleanup(): Promise<void>;
+}