@cosmocoder/mcp-web-docs 1.0.0

package/build/config.test.js

@@ -0,0 +1,165 @@
+import { isValidUrl, isValidPublicUrl, normalizeUrl, isGitHubUrl, isMarkdownPath, IGNORED_PATHS, RATE_LIMIT, QUEUE_OPTIONS, GITHUB_RATE_LIMIT, } from './config.js';
+describe('Configuration Utilities', () => {
+    describe('isValidUrl', () => {
+        it('should accept valid HTTP URLs', () => {
+            expect(isValidUrl('http://example.com')).toBe(true);
+            expect(isValidUrl('http://example.com:8080')).toBe(true);
+            expect(isValidUrl('http://example.com/path/to/page')).toBe(true);
+            expect(isValidUrl('http://example.com?query=1&other=2')).toBe(true);
+        });
+        it('should accept valid HTTPS URLs', () => {
+            expect(isValidUrl('https://example.com')).toBe(true);
+            expect(isValidUrl('https://subdomain.example.com')).toBe(true);
+            expect(isValidUrl('https://example.com/path')).toBe(true);
+        });
+        it('should reject non-HTTP protocols', () => {
+            expect(isValidUrl('ftp://example.com')).toBe(false);
+            expect(isValidUrl('file:///etc/passwd')).toBe(false);
+            expect(isValidUrl('javascript:alert(1)')).toBe(false);
+            expect(isValidUrl('data:text/html,<h1>Test</h1>')).toBe(false);
+        });
+        it('should reject invalid URLs', () => {
+            expect(isValidUrl('not-a-url')).toBe(false);
+            expect(isValidUrl('')).toBe(false);
+            expect(isValidUrl('example.com')).toBe(false);
+            expect(isValidUrl('://missing-protocol.com')).toBe(false);
+        });
+        it('should handle edge cases', () => {
+            expect(isValidUrl('http://localhost')).toBe(true);
+            expect(isValidUrl('http://127.0.0.1')).toBe(true);
+            expect(isValidUrl('http://[::1]')).toBe(true);
+        });
+    });
+    describe('isValidPublicUrl', () => {
+        it('should accept valid public URLs', () => {
+            expect(isValidPublicUrl('https://example.com')).toBe(true);
+            expect(isValidPublicUrl('https://docs.microsoft.com')).toBe(true);
+            expect(isValidPublicUrl('https://developer.mozilla.org')).toBe(true);
+        });
+        it('should reject private network URLs by default', () => {
+            expect(isValidPublicUrl('http://localhost')).toBe(false);
+            expect(isValidPublicUrl('http://127.0.0.1')).toBe(false);
+            expect(isValidPublicUrl('http://10.0.0.1')).toBe(false);
+            expect(isValidPublicUrl('http://192.168.1.1')).toBe(false);
+            expect(isValidPublicUrl('http://172.16.0.1')).toBe(false);
+        });
+        it('should allow private URLs when allowPrivate is true', () => {
+            expect(isValidPublicUrl('http://localhost', true)).toBe(true);
+            expect(isValidPublicUrl('http://192.168.1.1', true)).toBe(true);
+        });
+        it('should reject invalid URLs regardless of allowPrivate', () => {
+            expect(isValidPublicUrl('not-a-url', true)).toBe(false);
+            expect(isValidPublicUrl('ftp://example.com', true)).toBe(false);
+        });
+        it('should block cloud metadata endpoints', () => {
+            expect(isValidPublicUrl('http://169.254.169.254')).toBe(false);
+            expect(isValidPublicUrl('http://metadata.google.internal')).toBe(false);
+        });
+    });
+    describe('normalizeUrl', () => {
+        it('should remove trailing slashes', () => {
+            expect(normalizeUrl('https://example.com/')).toBe('https://example.com');
+            expect(normalizeUrl('https://example.com/path/')).toBe('https://example.com/path');
+        });
+        it('should preserve URL components', () => {
+            expect(normalizeUrl('https://example.com/path?query=1')).toBe('https://example.com/path?query=1');
+            expect(normalizeUrl('https://example.com:8080/path')).toBe('https://example.com:8080/path');
+        });
+        it('should handle URLs without trailing slash', () => {
+            expect(normalizeUrl('https://example.com')).toBe('https://example.com');
+            expect(normalizeUrl('https://example.com/path')).toBe('https://example.com/path');
+        });
+        it('should throw for invalid URLs', () => {
+            expect(() => normalizeUrl('not-a-url')).toThrow('Invalid URL');
+            expect(() => normalizeUrl('')).toThrow('Invalid URL');
+        });
+        it('should handle URLs with fragments', () => {
+            expect(normalizeUrl('https://example.com/page#section')).toBe('https://example.com/page#section');
+        });
+    });
+    describe('isGitHubUrl', () => {
+        it('should identify GitHub URLs', () => {
+            expect(isGitHubUrl('https://github.com/user/repo')).toBe(true);
+            expect(isGitHubUrl('https://github.com/user/repo/blob/main/README.md')).toBe(true);
+            expect(isGitHubUrl('https://github.com')).toBe(true);
+        });
+        it('should not identify non-GitHub URLs', () => {
+            expect(isGitHubUrl('https://gitlab.com/user/repo')).toBe(false);
+            expect(isGitHubUrl('https://bitbucket.org/user/repo')).toBe(false);
+            expect(isGitHubUrl('https://example.com')).toBe(false);
+        });
+        it('should handle GitHub-related but non-GitHub URLs', () => {
+            expect(isGitHubUrl('https://github.io')).toBe(false);
+            expect(isGitHubUrl('https://user.github.io')).toBe(false);
+            expect(isGitHubUrl('https://raw.githubusercontent.com/user/repo/main/file')).toBe(false);
+        });
+        it('should return false for invalid URLs', () => {
+            expect(isGitHubUrl('not-a-url')).toBe(false);
+            expect(isGitHubUrl('')).toBe(false);
+        });
+    });
+    describe('isMarkdownPath', () => {
+        it('should identify markdown file extensions', () => {
+            expect(isMarkdownPath('README.md')).toBe(true);
+            expect(isMarkdownPath('docs/guide.md')).toBe(true);
+            expect(isMarkdownPath('CHANGELOG.MD')).toBe(true);
+            expect(isMarkdownPath('file.mdx')).toBe(true);
+            expect(isMarkdownPath('file.markdown')).toBe(true);
+        });
+        it('should not identify non-markdown paths', () => {
+            expect(isMarkdownPath('file.txt')).toBe(false);
+            expect(isMarkdownPath('file.html')).toBe(false);
+            expect(isMarkdownPath('file.js')).toBe(false);
+            expect(isMarkdownPath('markdown')).toBe(false);
+        });
+        it('should handle paths without extensions', () => {
+            expect(isMarkdownPath('README')).toBe(false);
+            expect(isMarkdownPath('docs/guide')).toBe(false);
+        });
+        it('should handle edge cases', () => {
+            expect(isMarkdownPath('.md')).toBe(true);
+            expect(isMarkdownPath('file.md.bak')).toBe(false);
+            expect(isMarkdownPath('file.md/path')).toBe(false);
+        });
+    });
+    describe('Constants', () => {
+        describe('IGNORED_PATHS', () => {
+            it('should contain common non-content paths', () => {
+                expect(IGNORED_PATHS).toContain('favicon.ico');
+                expect(IGNORED_PATHS).toContain('robots.txt');
+                expect(IGNORED_PATHS).toContain('assets/');
+                expect(IGNORED_PATHS).toContain('node_modules/');
+            });
+            it('should be an array of strings', () => {
+                expect(Array.isArray(IGNORED_PATHS)).toBe(true);
+                IGNORED_PATHS.forEach((path) => {
+                    expect(typeof path).toBe('string');
+                });
+            });
+        });
+        describe('RATE_LIMIT', () => {
+            it('should have reasonable default values', () => {
+                expect(RATE_LIMIT.maxRequests).toBeGreaterThan(0);
+                expect(RATE_LIMIT.timeWindow).toBeGreaterThan(0);
+                expect(RATE_LIMIT.minDelay).toBeGreaterThanOrEqual(0);
+            });
+        });
+        describe('QUEUE_OPTIONS', () => {
+            it('should have retry configuration', () => {
+                expect(QUEUE_OPTIONS.maxRequestRetries).toBeGreaterThanOrEqual(0);
+                expect(QUEUE_OPTIONS.retryDelay).toBeGreaterThan(0);
+                expect(QUEUE_OPTIONS.maxRequestsPerCrawl).toBeGreaterThan(0);
+            });
+        });
+        describe('GITHUB_RATE_LIMIT', () => {
+            it('should have different limits for authenticated vs unauthenticated', () => {
+                expect(GITHUB_RATE_LIMIT.authenticated.maxRequests).toBeGreaterThan(GITHUB_RATE_LIMIT.unauthenticated.maxRequests);
+            });
+            it('should have time windows defined', () => {
+                expect(GITHUB_RATE_LIMIT.authenticated.timeWindow).toBeGreaterThan(0);
+                expect(GITHUB_RATE_LIMIT.unauthenticated.timeWindow).toBeGreaterThan(0);
+            });
+        });
+    });
+});
+//# sourceMappingURL=config.test.js.map

package/build/config.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.test.js","sourceRoot":"","sources":["../src/config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,cAAc,EACd,aAAa,EACb,UAAU,EACV,aAAa,EACb,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,MAAM,CAAC,UAAU,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjE,MAAM,CAAC,UAAU,CAAC,oCAAoC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClE,MAAM,CAAC,gBAAgB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9D,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,CAAC,gBAAgB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/D,MAAM,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACzE,MAAM,CAAC,YAAY,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClG,MAAM,CAAC,YAAY,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACxE,MAAM,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAC/D,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,WAAW,CAAC,kDAAkD,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnF,MAAM,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChE,MAAM,CAAC,WAAW,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1D,MAAM,CAAC,WAAW,CAAC,uDAAuD,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/C,MAAM,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChD,MAAM,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;YAC7B,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;gBACjD,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBAC/C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;gBAC9C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAC3C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YACnD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;gBACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChD,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC7B,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACrC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;YAC1B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;gBAC/C,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBACjD,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;YAC7B,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;gBACzC,MAAM,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;gBAClE,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBACpD,MAAM,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;YACjC,EAAE,CAAC,mEAAmE,EAAE,GAAG,EAAE;gBAC3E,MAAM,CAAC,iBAAiB,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,iBAAiB,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;YACrH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;gBAC1C,MAAM,CAAC,iBAAiB,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBACtE,MAAM,CAAC,iBAAiB,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/build/crawler/auth.d.ts

@@ -0,0 +1,128 @@
+import { Browser, BrowserContext } from 'playwright';
+import { type LoginPageDetectionResult } from '../util/security.js';
+/** Supported browser types */
+export type BrowserType = 'chromium' | 'chrome' | 'firefox' | 'webkit' | 'edge';
+/**
+ * Detect the user's default browser from OS settings using the default-browser package
+ * https://github.com/sindresorhus/default-browser
+ */
+export declare function detectDefaultBrowser(): Promise<BrowserType>;
+/** Authentication options for crawling */
+export interface AuthOptions {
+    /** Whether authentication is required */
+    requiresAuth?: boolean;
+    /** Browser to use for authentication */
+    browser?: BrowserType;
+    /** URL pattern that indicates successful login (regex) */
+    loginSuccessPattern?: string;
+    /** CSS selector that appears after successful login */
+    loginSuccessSelector?: string;
+    /** Login page URL (if different from main URL) */
+    loginUrl?: string;
+    /** Timeout for waiting for login (seconds) */
+    loginTimeoutSecs?: number;
+}
+/**
+ * Manages authentication sessions for crawling protected pages.
+ * Opens a visible browser for user to login, then saves the session for reuse.
+ */
+export declare class AuthManager {
+    private sessionsDir;
+    private activeBrowser;
+    private activeContext;
+    constructor(dataDir: string);
+    initialize(): Promise<void>;
+    /**
+     * Get the session file path for a domain with path traversal protection
+     */
+    private getSessionPath;
+    /**
+     * Check if we have a saved session for a domain
+     */
+    hasSession(url: string): Promise<boolean>;
+    /**
+     * Load a saved session for a domain
+     * Sessions are encrypted at rest and validated on load
+     */
+    loadSession(url: string): Promise<string | null>;
+    /**
+     * Save a session for a domain with encryption
+     * The storage state is encrypted before being written to disk
+     */
+    private saveSession;
+    /**
+     * Clear a saved session for a domain
+     */
+    clearSession(url: string): Promise<void>;
+    /**
+     * Validate that a stored session is still valid by making a test request.
+     * This detects expired sessions by checking for:
+     * 1. Redirects to login/auth pages
+     * 2. Login page content in the response
+     *
+     * @param url - The protected URL to validate against
+     * @param browserType - Browser type to use for validation
+     * @returns Validation result indicating if session is still valid
+     */
+    validateSession(url: string, browserType?: BrowserType): Promise<{
+        isValid: boolean;
+        reason?: string;
+        loginDetection?: LoginPageDetectionResult;
+        finalUrl?: string;
+    }>;
+    /**
+     * Validate session and throw SessionExpiredError if expired.
+     * This is a convenience method for use in crawling workflows.
+     *
+     * @param url - The protected URL to validate against
+     * @param browserType - Browser type to use for validation
+     * @throws SessionExpiredError if the session has expired
+     */
+    validateSessionOrThrow(url: string, browserType?: BrowserType): Promise<void>;
+    /**
+     * Get the Playwright browser launcher for a browser type
+     */
+    private getBrowserLauncher;
+    /**
+     * Get launch options for a browser type
+     *
+     * Note: Firefox cannot use the installed browser - Playwright requires its bundled
+     * Firefox (Nightly) which includes the Juggler automation protocol.
+     * Chrome and Edge can use the installed browser via the 'channel' option.
+     */
+    private getLaunchOptions;
+    /**
+     * Open a visible browser for user to login interactively.
+     * Returns the storage state (cookies/localStorage) after successful login.
+     *
+     * Opens a fresh browser instance for the user to login manually.
+     * Note: We don't use the user's existing browser profile because:
+     * 1. Firefox locks profiles when already open
+     * 2. Playwright uses its own bundled browsers, not system browsers
+     */
+    performInteractiveLogin(url: string, options?: AuthOptions): Promise<string>;
+    /**
+     * Login using a fresh browser instance (for manual login)
+     */
+    private loginWithFreshBrowser;
+    /**
+     * Wait for the user to complete login.
+     *
+     * Detection methods (in order of priority):
+     * 1. If successPattern is provided: wait for URL to match the regex
+     * 2. If successSelector is provided: wait for the CSS selector to appear
+     * 3. Default: poll for common login success indicators (logout button, user menu, URL change)
+     */
+    private waitForLogin;
+    /**
+     * Create a browser context with saved authentication
+     */
+    createAuthenticatedContext(url: string, browserType?: BrowserType): Promise<{
+        browser: Browser;
+        context: BrowserContext;
+    } | null>;
+    /**
+     * Clean up any active browser instances
+     */
+    cleanup(): Promise<void>;
+}