@cosmocoder/mcp-web-docs 1.0.0

package/build/crawler/queue-manager.test.js

@@ -0,0 +1,509 @@
+import { QueueManager } from './queue-manager.js';
+const { mockRequestQueue, mockDataset } = vi.hoisted(() => ({
+    mockRequestQueue: {
+        drop: vi.fn().mockResolvedValue(undefined),
+        addRequest: vi.fn().mockResolvedValue(undefined),
+        getInfo: vi.fn().mockResolvedValue({
+            pendingRequestCount: 5,
+            handledRequestCount: 10,
+            totalRequestCount: 15,
+        }),
+    },
+    mockDataset: {
+        drop: vi.fn().mockResolvedValue(undefined),
+        pushData: vi.fn().mockResolvedValue(undefined),
+    },
+}));
+vi.mock('crawlee', () => ({
+    RequestQueue: {
+        open: vi.fn().mockResolvedValue(mockRequestQueue),
+    },
+    Dataset: {
+        open: vi.fn().mockResolvedValue(mockDataset),
+    },
+    EnqueueStrategy: {
+        SameDomain: 'same-domain',
+    },
+}));
+describe('QueueManager', () => {
+    let queueManager;
+    beforeEach(() => {
+        vi.clearAllMocks();
+        queueManager = new QueueManager();
+    });
+    describe('initialize', () => {
+        it('should initialize with a URL', async () => {
+            await queueManager.initialize('https://example.com/docs');
+            expect(mockRequestQueue.drop).toHaveBeenCalled();
+            expect(mockRequestQueue.addRequest).toHaveBeenCalledWith({
+                url: 'https://example.com/docs',
+                uniqueKey: '/docs',
+            });
+        });
+        it('should generate correct unique key from URL', async () => {
+            await queueManager.initialize('https://example.com/path/to/page?query=1');
+            expect(mockRequestQueue.addRequest).toHaveBeenCalledWith({
+                url: 'https://example.com/path/to/page?query=1',
+                uniqueKey: '/path/to/page?query=1',
+            });
+        });
+        it('should clear existing dataset', async () => {
+            await queueManager.initialize('https://example.com');
+            expect(mockDataset.drop).toHaveBeenCalled();
+        });
+    });
+    describe('handleQueueAndLinks', () => {
+        const mockLog = {
+            info: vi.fn(),
+            debug: vi.fn(),
+            warning: vi.fn(),
+            error: vi.fn(),
+        };
+        const mockRule = {
+            type: 'default',
+            extractor: { extractContent: vi.fn() },
+            detect: vi.fn().mockResolvedValue(true),
+        };
+        beforeEach(async () => {
+            await queueManager.initialize('https://example.com');
+        });
+        it('should log queue status', async () => {
+            const mockEnqueueLinks = vi.fn().mockResolvedValue({
+                processedRequests: [],
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            expect(mockLog.info).toHaveBeenCalledWith('Queue status:', {
+                pendingCount: 5,
+                handledCount: 10,
+                totalCount: 15,
+            });
+        });
+        it('should enqueue links with same-domain strategy', async () => {
+            const mockEnqueueLinks = vi.fn().mockResolvedValue({
+                processedRequests: [{ uniqueKey: '/page1' }, { uniqueKey: '/page2' }],
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            expect(mockEnqueueLinks).toHaveBeenCalledWith(expect.objectContaining({
+                strategy: 'same-domain',
+            }));
+        });
+        it('should use link selectors from rule when provided', async () => {
+            const ruleWithSelectors = {
+                ...mockRule,
+                linkSelectors: ['.nav a', '.content a'],
+            };
+            const mockEnqueueLinks = vi.fn().mockResolvedValue({
+                processedRequests: [],
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, ruleWithSelectors);
+            expect(mockEnqueueLinks).toHaveBeenCalledWith(expect.objectContaining({
+                selector: '.nav a, .content a',
+            }));
+        });
+        it('should log enqueued links count', async () => {
+            const mockEnqueueLinks = vi.fn().mockResolvedValue({
+                processedRequests: [{ uniqueKey: '/page1' }, { uniqueKey: '/page2' }],
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            expect(mockLog.info).toHaveBeenCalledWith('Enqueued links:', {
+                processedCount: 2,
+                urls: ['/page1', '/page2'],
+            });
+        });
+        it('should transform request URLs to use pathname as unique key', async () => {
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            expect(capturedOptions).not.toBeNull();
+            const transformFn = capturedOptions.transformRequestFunction;
+            expect(transformFn).toBeDefined();
+            if (transformFn) {
+                const transformed = transformFn({
+                    url: 'https://example.com/new/page?param=1',
+                    uniqueKey: 'original',
+                });
+                expect(transformed).toEqual(expect.objectContaining({
+                    url: 'https://example.com/new/page?param=1',
+                    uniqueKey: '/new/page?param=1',
+                }));
+            }
+        });
+        it('should skip anchor links with hash fragments', async () => {
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            expect(capturedOptions).not.toBeNull();
+            const transformFn = capturedOptions.transformRequestFunction;
+            expect(transformFn).toBeDefined();
+            if (transformFn) {
+                // Test that anchor links are filtered out (return false)
+                const anchorResult = transformFn({
+                    url: 'https://example.com/page#section',
+                    uniqueKey: 'original',
+                });
+                expect(anchorResult).toBe(false);
+            }
+        });
+        it('should strip hash from URL when returning transformed request', async () => {
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                // URL without hash should be returned with clean URL
+                const transformed = transformFn({
+                    url: 'https://example.com/page',
+                    uniqueKey: 'original',
+                });
+                expect(transformed).toEqual(expect.objectContaining({
+                    url: 'https://example.com/page',
+                    uniqueKey: '/page',
+                }));
+            }
+        });
+    });
+    describe('path prefix filtering', () => {
+        const mockLog = {
+            info: vi.fn(),
+            debug: vi.fn(),
+            warning: vi.fn(),
+            error: vi.fn(),
+        };
+        const mockRule = {
+            type: 'default',
+            extractor: { extractContent: vi.fn() },
+            detect: vi.fn().mockResolvedValue(true),
+        };
+        it('should initialize with path prefix', async () => {
+            await queueManager.initialize('https://example.com/docs/api', '/docs/api');
+            // The path prefix should be stored (we can verify through transform function behavior)
+            expect(queueManager.getFilteredByPathCount()).toBe(0);
+        });
+        it('should allow URLs within path prefix', async () => {
+            await queueManager.initialize('https://example.com/docs', '/docs');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                // Exact match
+                const exactMatch = transformFn({
+                    url: 'https://example.com/docs',
+                    uniqueKey: 'original',
+                });
+                expect(exactMatch).not.toBe(false);
+                // Subpath
+                const subpath = transformFn({
+                    url: 'https://example.com/docs/api/v2',
+                    uniqueKey: 'original',
+                });
+                expect(subpath).not.toBe(false);
+            }
+        });
+        it('should filter URLs outside path prefix', async () => {
+            await queueManager.initialize('https://example.com/docs', '/docs');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                // Different path entirely
+                const differentPath = transformFn({
+                    url: 'https://example.com/blog/post',
+                    uniqueKey: 'original',
+                });
+                expect(differentPath).toBe(false);
+            }
+        });
+        it('should not match paths that only start with prefix string', async () => {
+            await queueManager.initialize('https://example.com/docs', '/docs');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                // /documentation starts with /docs but is NOT a subpath of /docs
+                const similarButDifferent = transformFn({
+                    url: 'https://example.com/documentation',
+                    uniqueKey: 'original',
+                });
+                expect(similarButDifferent).toBe(false);
+            }
+        });
+        it('should track filtered URL count', async () => {
+            await queueManager.initialize('https://example.com/docs', '/docs');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                // Filter some URLs
+                transformFn({ url: 'https://example.com/blog', uniqueKey: 'a' });
+                transformFn({ url: 'https://example.com/about', uniqueKey: 'b' });
+            }
+            expect(queueManager.getFilteredByPathCount()).toBe(2);
+        });
+        it('should strip hash from initial URL', async () => {
+            await queueManager.initialize('https://example.com/docs#intro');
+            expect(mockRequestQueue.addRequest).toHaveBeenCalledWith({
+                url: 'https://example.com/docs',
+                uniqueKey: '/docs',
+            });
+        });
+    });
+    describe('hostname filtering', () => {
+        const mockLog = {
+            info: vi.fn(),
+            debug: vi.fn(),
+            warning: vi.fn(),
+            error: vi.fn(),
+        };
+        const mockRule = {
+            type: 'default',
+            extractor: { extractContent: vi.fn() },
+            detect: vi.fn().mockResolvedValue(true),
+        };
+        it('should allow URLs with exact hostname match', async () => {
+            await queueManager.initialize('https://docs.example.com/api');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                const result = transformFn({
+                    url: 'https://docs.example.com/other',
+                    uniqueKey: 'original',
+                });
+                expect(result).not.toBe(false);
+            }
+        });
+        it('should allow URLs with subdomain of starting hostname', async () => {
+            await queueManager.initialize('https://docs.example.com/api');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                // api.docs.example.com is a subdomain of docs.example.com - should be allowed
+                const result = transformFn({
+                    url: 'https://api.docs.example.com/endpoint',
+                    uniqueKey: 'original',
+                });
+                expect(result).not.toBe(false);
+            }
+        });
+        it('should filter URLs with sibling subdomain', async () => {
+            await queueManager.initialize('https://docs.example.com/api');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                // python.example.com is a sibling subdomain, not a subdomain of docs.example.com
+                const result = transformFn({
+                    url: 'https://python.example.com/guide',
+                    uniqueKey: 'original',
+                });
+                expect(result).toBe(false);
+            }
+        });
+        it('should filter URLs with parent domain', async () => {
+            await queueManager.initialize('https://docs.example.com/api');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                // example.com is the parent domain of docs.example.com
+                const result = transformFn({
+                    url: 'https://example.com/home',
+                    uniqueKey: 'original',
+                });
+                expect(result).toBe(false);
+            }
+        });
+        it('should track filtered hostname count', async () => {
+            await queueManager.initialize('https://docs.example.com/api');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                // Filter some URLs with wrong hostnames
+                transformFn({ url: 'https://python.example.com/a', uniqueKey: 'a' });
+                transformFn({ url: 'https://other.example.com/b', uniqueKey: 'b' });
+            }
+            expect(queueManager.getFilteredByHostnameCount()).toBe(2);
+        });
+        it('should handle case-insensitive hostname matching', async () => {
+            await queueManager.initialize('https://Docs.Example.Com/api');
+            let capturedOptions = null;
+            const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
+                capturedOptions = options;
+                return Promise.resolve({ processedRequests: [] });
+            });
+            await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
+            const transformFn = capturedOptions.transformRequestFunction;
+            if (transformFn) {
+                const result = transformFn({
+                    url: 'https://docs.example.com/other',
+                    uniqueKey: 'original',
+                });
+                expect(result).not.toBe(false);
+            }
+        });
+    });
+    describe('addResult and processBatch', () => {
+        beforeEach(async () => {
+            await queueManager.initialize('https://example.com');
+        });
+        it('should add results', () => {
+            const result = {
+                url: 'https://example.com/page',
+                path: '/page',
+                content: 'Test content',
+                title: 'Test Page',
+            };
+            queueManager.addResult(result);
+            expect(queueManager.hasEnoughResults()).toBe(false);
+        });
+        it('should process batch and return results', async () => {
+            const result = {
+                url: 'https://example.com/page',
+                path: '/page',
+                content: 'Test content',
+                title: 'Test Page',
+            };
+            queueManager.addResult(result);
+            const processed = await queueManager.processBatch();
+            expect(processed).toHaveLength(1);
+            expect(processed[0]).toEqual(result);
+            expect(mockDataset.pushData).toHaveBeenCalled();
+        });
+        it('should return empty array when no results', async () => {
+            const processed = await queueManager.processBatch();
+            expect(processed).toHaveLength(0);
+        });
+        it('should clear results after processing', async () => {
+            queueManager.addResult({
+                url: 'https://example.com/page',
+                path: '/page',
+                content: 'Test',
+                title: 'Test',
+            });
+            await queueManager.processBatch();
+            const secondBatch = await queueManager.processBatch();
+            expect(secondBatch).toHaveLength(0);
+        });
+        it('should push data to dataset in chunks', async () => {
+            // Add 7 results
+            for (let i = 0; i < 7; i++) {
+                queueManager.addResult({
+                    url: `https://example.com/page${i}`,
+                    path: `/page${i}`,
+                    content: `Content ${i}`,
+                    title: `Page ${i}`,
+                });
+            }
+            await queueManager.processBatch();
+            // Should be pushed in chunks of 5
+            expect(mockDataset.pushData).toHaveBeenCalledTimes(2);
+        });
+    });
+    describe('hasEnoughResults', () => {
+        beforeEach(async () => {
+            await queueManager.initialize('https://example.com');
+        });
+        it('should return false when below batch size', () => {
+            for (let i = 0; i < 10; i++) {
+                queueManager.addResult({
+                    url: `https://example.com/page${i}`,
+                    path: `/page${i}`,
+                    content: `Content ${i}`,
+                    title: `Page ${i}`,
+                });
+            }
+            expect(queueManager.hasEnoughResults()).toBe(false);
+        });
+        it('should return true when at batch size', () => {
+            for (let i = 0; i < 20; i++) {
+                queueManager.addResult({
+                    url: `https://example.com/page${i}`,
+                    path: `/page${i}`,
+                    content: `Content ${i}`,
+                    title: `Page ${i}`,
+                });
+            }
+            expect(queueManager.hasEnoughResults()).toBe(true);
+        });
+    });
+    describe('getRequestQueue', () => {
+        it('should return null before initialization', () => {
+            expect(queueManager.getRequestQueue()).toBeNull();
+        });
+        it('should return request queue after initialization', async () => {
+            await queueManager.initialize('https://example.com');
+            expect(queueManager.getRequestQueue()).toBe(mockRequestQueue);
+        });
+    });
+    describe('cleanup', () => {
+        it('should clear results and drop queue', async () => {
+            await queueManager.initialize('https://example.com');
+            queueManager.addResult({
+                url: 'https://example.com/page',
+                path: '/page',
+                content: 'Test',
+                title: 'Test',
+            });
+            await queueManager.cleanup();
+            expect(mockRequestQueue.drop).toHaveBeenCalled();
+            expect(queueManager.getRequestQueue()).toBeNull();
+            // Results should be cleared
+            const processed = await queueManager.processBatch();
+            expect(processed).toHaveLength(0);
+        });
+        it('should handle cleanup errors gracefully', async () => {
+            await queueManager.initialize('https://example.com');
+            mockRequestQueue.drop.mockRejectedValueOnce(new Error('Drop failed'));
+            // Should not throw
+            await expect(queueManager.cleanup()).resolves.toBeUndefined();
+        });
+    });
+});
+//# sourceMappingURL=queue-manager.test.js.map

package/build/crawler/queue-manager.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue-manager.test.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1D,gBAAgB,EAAE;QAChB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAChD,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;YACjC,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,EAAE;YACvB,iBAAiB,EAAE,EAAE;SACtB,CAAC;KACH;IACD,WAAW,EAAE;QACX,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;KAC/C;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,YAAY,EAAE;QACZ,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;KAClD;IACD,OAAO,EAAE;QACP,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC;KAC7C;IACD,eAAe,EAAE;QACf,UAAU,EAAE,aAAa;KAC1B;CACF,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAI,YAA0B,CAAC;IAE/B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAE1D,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0BAA0B;gBAC/B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,0CAA0C,CAAC,CAAC;YAE1E,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0CAA0C;gBAC/C,SAAS,EAAE,uBAAuB;aACnC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC9C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,eAAe,EAAE;gBACzD,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE,EAAE;gBAChB,UAAU,EAAE,EAAE;aACf,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,aAAa;aACxB,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,iBAAiB,GAAsB;gBAC3C,GAAG,QAAQ;gBACX,aAAa,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC;aACxC,CAAC;YAEF,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC;YAErF,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,oBAAoB;aAC/B,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,iBAAiB,EAAE;gBAC3D,cAAc,EAAE,CAAC;gBACjB,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,mBAAmB;iBAC/B,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,yDAAyD;gBACzD,MAAM,YAAY,GAAG,WAAW,CAAC;oBAC/B,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,qDAAqD;gBACrD,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,OAAO;iBACnB,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;YAE3E,uFAAuF;YACvF,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,cAAc;gBACd,MAAM,UAAU,GAAG,WAAW,CAAC;oBAC7B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEnC,UAAU;gBACV,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,iCAAiC;oBACtC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,0BAA0B;gBAC1B,MAAM,aAAa,GAAG,WAAW,CAAC;oBAChC,GAAG,EAAE,+BAA+B;oBACpC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iEAAiE;gBACjE,MAAM,mBAAmB,GAAG,WAAW,CAAC;oBACtC,GAAG,EAAE,mCAAmC;oBACxC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,mBAAmB;gBACnB,WAAW,CAAC,EAAE,GAAG,EAAE,0BAA0B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBACtG,WAAW,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YACzG,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,gCAAgC,CAAC,CAAC;YAEhE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0BAA0B;gBAC/B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,8EAA8E;gBAC9E,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,uCAAuC;oBAC5C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iFAAiF;gBACjF,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,uDAAuD;gBACvD,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,wCAAwC;gBACxC,WAAW,CAAC,EAAE,GAAG,EAAE,8BAA8B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBAC1G,WAAW,CAAC,EAAE,GAAG,EAAE,6BAA6B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YAC3G,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,0BAA0B,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAC5B,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEtD,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,gBAAgB;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAElC,kCAAkC;YAClC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;YAE7B,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;YAElD,4BAA4B;YAC5B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAEtE,mBAAmB;YACnB,MAAM,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/site-rules.d.ts

@@ -0,0 +1,11 @@
+import { Log } from 'crawlee';
+import { Page } from 'playwright';
+import { ContentExtractor } from './content-extractor-types.js';
+export interface SiteDetectionRule {
+    type: string;
+    extractor: ContentExtractor;
+    detect: (page: Page) => Promise<boolean>;
+    prepare?: (page: Page, log: Log) => Promise<void>;
+    linkSelectors?: string[];
+}
+export declare const siteRules: SiteDetectionRule[];

package/build/crawler/site-rules.js

@@ -0,0 +1,104 @@
+import { contentExtractors } from './content-extractors.js';
+export const siteRules = [
+    {
+        type: 'storybook',
+        extractor: contentExtractors.storybook,
+        detect: async (page) => {
+            return page.evaluate(() => {
+                return !!(document.querySelector('#storybook-root, .sbdocs, [data-nodetype="root"]') ||
+                    document.querySelector('meta[name="storybook-version"]') ||
+                    document.baseURI?.includes('path=/docs/') ||
+                    document.baseURI?.includes('path=/story/') ||
+                    window.__STORYBOOK_CLIENT_API__);
+            });
+        },
+        prepare: async (page, log) => {
+            await Promise.all([
+                page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => log.debug('Network idle timeout - continuing anyway')),
+                page
+                    .waitForSelector('.sbdocs-content, #docs-root, .docs-story, [class*="story-"]', {
+                    timeout: 5000,
+                })
+                    .catch(() => log.debug('No Storybook content found in main page')),
+            ]);
+            // Wait for sidebar to be ready
+            await page.waitForSelector('[class*="sidebar"]', { timeout: 5000 }).catch(() => log.debug('No sidebar found'));
+            // First expand all section buttons
+            await page.evaluate(() => {
+                const buttons = Array.from(document.querySelectorAll('button.sidebar-subheading-action'));
+                buttons.forEach((button) => button.click());
+            });
+            // Wait for any new content to appear
+            await page.waitForTimeout(500);
+            // Then expand any remaining collapsed sections
+            await page.evaluate(() => {
+                const expandButtons = Array.from(document.querySelectorAll('[aria-expanded="false"]'));
+                expandButtons.forEach((button) => button.click());
+            });
+            // Wait for all animations and content updates to complete
+            await page.waitForTimeout(1000);
+            // Scroll to bottom to trigger lazy loading of ArgTypes tables
+            await page.evaluate(async () => {
+                // Find the content iframe
+                const iframe = document.querySelector('iframe');
+                const contentDoc = iframe?.contentDocument || document;
+                // Scroll to bottom to load all content
+                const scrollContainer = contentDoc.querySelector('.sbdocs-content, #docs-root, body');
+                if (scrollContainer) {
+                    scrollContainer.scrollTo(0, scrollContainer.scrollHeight);
+                    await new Promise((resolve) => setTimeout(resolve, 500));
+                    scrollContainer.scrollTo(0, 0);
+                }
+            });
+            // Click on "Show code" buttons to reveal code examples
+            await page.evaluate(() => {
+                const showCodeButtons = Array.from(document.querySelectorAll('button')).filter((btn) => btn.textContent?.toLowerCase().includes('show code'));
+                showCodeButtons.slice(0, 3).forEach((btn) => btn.click()); // Limit to first 3
+            });
+            await page.waitForTimeout(500);
+            // Expand ArgTypes table rows and "Show more" buttons
+            await page.evaluate(async () => {
+                const iframe = document.querySelector('iframe');
+                const contentDoc = iframe?.contentDocument || document;
+                // Click expand buttons in args table
+                const expandBtns = contentDoc.querySelectorAll('[class*="argstable"] button[aria-expanded="false"], ' + '[class*="argtable"] button[aria-expanded="false"]');
+                expandBtns.forEach((btn) => btn.click());
+                // Click all "Show X more..." buttons to reveal full type lists
+                const showMoreBtns = Array.from(contentDoc.querySelectorAll('button')).filter((btn) => btn.textContent?.includes('Show') && btn.textContent?.includes('more'));
+                for (const btn of showMoreBtns) {
+                    btn.click();
+                    await new Promise((resolve) => setTimeout(resolve, 100));
+                }
+            });
+            await page.waitForTimeout(500);
+            // Log the number of links found for debugging
+            const linkCount = await page.evaluate(() => {
+                return document.querySelectorAll('.sidebar-item a, [data-nodetype="story"] a, [data-nodetype="document"] a').length;
+            });
+            log.debug(`Found ${linkCount} sidebar links after expansion`);
+        },
+        linkSelectors: [
+            '.sidebar-item a',
+            '[data-nodetype="root"] a',
+            '[data-nodetype="group"] a',
+            '[data-nodetype="document"] a',
+            '[data-nodetype="story"] a',
+            '[data-item-id] a',
+        ],
+    },
+    {
+        type: 'github',
+        extractor: contentExtractors.github,
+        detect: async (page) => {
+            return page.evaluate(() => {
+                return (window.location.hostname.includes('github.io') && document.querySelector('.markdown-body, .site-footer, .page-header') !== null);
+            });
+        },
+    },
+    {
+        type: 'default',
+        extractor: contentExtractors.default,
+        detect: async () => true,
+    },
+];
+//# sourceMappingURL=site-rules.js.map