@lobehub/chat 1.124.0 → 1.124.2

package/packages/web-crawler/src/crawImpl/__tests__/tavily.test.ts

@@ -0,0 +1,359 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../../utils/errorType';
+import { tavily } from '../tavily';
+
+// Mock dependencies
+vi.mock('../../utils/withTimeout', () => ({
+  DEFAULT_TIMEOUT: 30000,
+  withTimeout: vi.fn(),
+}));
+
+describe('tavily crawler', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    delete process.env.TAVILY_API_KEY;
+    delete process.env.TAVILY_EXTRACT_DEPTH;
+  });
+
+  it('should successfully crawl content with API key', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        base_url: 'https://api.tavily.com',
+        response_time: 1.5,
+        results: [
+          {
+            url: 'https://example.com',
+            raw_content:
+              'This is a test raw content with sufficient length to pass validation. '.repeat(3),
+            images: ['https://example.com/image1.jpg', 'https://example.com/image2.jpg'],
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await tavily('https://example.com', { filterOptions: {} });
+
+    expect(result).toEqual({
+      content: 'This is a test raw content with sufficient length to pass validation. '.repeat(3),
+      contentType: 'text',
+      length: 'This is a test raw content with sufficient length to pass validation. '.repeat(3)
+        .length,
+      siteName: 'example.com',
+      title: 'example.com',
+      url: 'https://example.com',
+    });
+
+    expect(withTimeout).toHaveBeenCalledWith(expect.any(Promise), 30000);
+  });
+
+  it('should use custom extract depth when provided', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+    process.env.TAVILY_EXTRACT_DEPTH = 'advanced';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        base_url: 'https://api.tavily.com',
+        response_time: 2.1,
+        results: [
+          {
+            url: 'https://example.com',
+            raw_content: 'Advanced extraction content with more details. '.repeat(5),
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await tavily('https://example.com', { filterOptions: {} });
+
+    expect(withTimeout).toHaveBeenCalledWith(expect.any(Promise), 30000);
+  });
+
+  it('should handle missing API key', async () => {
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        base_url: 'https://api.tavily.com',
+        response_time: 1.2,
+        results: [
+          {
+            url: 'https://example.com',
+            raw_content: 'Test content with sufficient length. '.repeat(5),
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await tavily('https://example.com', { filterOptions: {} });
+
+    expect(withTimeout).toHaveBeenCalledWith(expect.any(Promise), 30000);
+  });
+
+  it('should return undefined when no results are returned', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        base_url: 'https://api.tavily.com',
+        response_time: 0.8,
+        results: [],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    const result = await tavily('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+    expect(consoleSpy).toHaveBeenCalledWith(
+      'Tavily API returned no results for URL:',
+      'https://example.com',
+    );
+
+    consoleSpy.mockRestore();
+  });
+
+  it('should return undefined for short content', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        base_url: 'https://api.tavily.com',
+        response_time: 1.1,
+        results: [
+          {
+            url: 'https://example.com',
+            raw_content: 'Short', // Content too short
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await tavily('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should return undefined when raw_content is missing', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        base_url: 'https://api.tavily.com',
+        response_time: 1.0,
+        results: [
+          {
+            url: 'https://example.com',
+            // raw_content is missing
+            images: ['https://example.com/image.jpg'],
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await tavily('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should throw PageNotFoundError for 404 status', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: false,
+      status: 404,
+      statusText: 'Not Found',
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await expect(tavily('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      PageNotFoundError,
+    );
+  });
+
+  it('should throw error for other HTTP errors', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: false,
+      status: 500,
+      statusText: 'Internal Server Error',
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await expect(tavily('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      'Tavily request failed with status 500: Internal Server Error',
+    );
+  });
+
+  it('should throw NetworkConnectionError for fetch failures', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(new Error('fetch failed'));
+
+    await expect(tavily('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      NetworkConnectionError,
+    );
+  });
+
+  it('should throw TimeoutError when request times out', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const timeoutError = new TimeoutError('Request timeout');
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(timeoutError);
+
+    await expect(tavily('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      TimeoutError,
+    );
+  });
+
+  it('should rethrow unknown errors', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const unknownError = new Error('Unknown error');
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(unknownError);
+
+    await expect(tavily('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      'Unknown error',
+    );
+  });
+
+  it('should return undefined when JSON parsing fails', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockRejectedValue(new Error('Invalid JSON')),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    const result = await tavily('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+    expect(consoleSpy).toHaveBeenCalled();
+
+    consoleSpy.mockRestore();
+  });
+
+  it('should use result URL when available', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        base_url: 'https://api.tavily.com',
+        response_time: 1.3,
+        results: [
+          {
+            url: 'https://redirected.example.com',
+            raw_content: 'Test content with sufficient length. '.repeat(5),
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await tavily('https://example.com', { filterOptions: {} });
+
+    expect(result?.url).toBe('https://redirected.example.com');
+  });
+
+  it('should fallback to original URL when result URL is missing', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        base_url: 'https://api.tavily.com',
+        response_time: 1.4,
+        results: [
+          {
+            raw_content: 'Test content with sufficient length. '.repeat(5),
+            // url is missing
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await tavily('https://example.com', { filterOptions: {} });
+
+    expect(result?.url).toBe('https://example.com');
+  });
+
+  it('should handle failed results in response', async () => {
+    process.env.TAVILY_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        base_url: 'https://api.tavily.com',
+        response_time: 1.6,
+        results: [],
+        failed_results: [
+          {
+            url: 'https://example.com',
+            error: 'Page not accessible',
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    const result = await tavily('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+    expect(consoleSpy).toHaveBeenCalledWith(
+      'Tavily API returned no results for URL:',
+      'https://example.com',
+    );
+
+    consoleSpy.mockRestore();
+  });
+});

package/packages/web-crawler/src/utils/__tests__/errorType.test.ts

@@ -0,0 +1,217 @@
+import { describe, expect, it } from 'vitest';
+
+import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../errorType';
+
+describe('errorType', () => {
+  describe('PageNotFoundError', () => {
+    it('should create error with correct message and name', () => {
+      const message = 'Page not found';
+      const error = new PageNotFoundError(message);
+
+      expect(error).toBeInstanceOf(Error);
+      expect(error).toBeInstanceOf(PageNotFoundError);
+      expect(error.message).toBe(message);
+      expect(error.name).toBe('PageNotFoundError');
+    });
+
+    it('should preserve stack trace', () => {
+      const error = new PageNotFoundError('Test message');
+      expect(error.stack).toBeDefined();
+      expect(typeof error.stack).toBe('string');
+    });
+
+    it('should be throwable and catchable', () => {
+      const message = 'Custom 404 message';
+
+      expect(() => {
+        throw new PageNotFoundError(message);
+      }).toThrow(PageNotFoundError);
+
+      expect(() => {
+        throw new PageNotFoundError(message);
+      }).toThrow(message);
+    });
+
+    it('should handle empty message', () => {
+      const error = new PageNotFoundError('');
+      expect(error.message).toBe('');
+      expect(error.name).toBe('PageNotFoundError');
+    });
+
+    it('should handle special characters in message', () => {
+      const specialMessage = 'Page not found: ëñçødéd tëxt & symbols!@#$%^&*()';
+      const error = new PageNotFoundError(specialMessage);
+      expect(error.message).toBe(specialMessage);
+      expect(error.name).toBe('PageNotFoundError');
+    });
+  });
+
+  describe('NetworkConnectionError', () => {
+    it('should create error with default message and correct name', () => {
+      const error = new NetworkConnectionError();
+
+      expect(error).toBeInstanceOf(Error);
+      expect(error).toBeInstanceOf(NetworkConnectionError);
+      expect(error.message).toBe('Network connection error');
+      expect(error.name).toBe('NetworkConnectionError');
+    });
+
+    it('should preserve stack trace', () => {
+      const error = new NetworkConnectionError();
+      expect(error.stack).toBeDefined();
+      expect(typeof error.stack).toBe('string');
+    });
+
+    it('should be throwable and catchable', () => {
+      expect(() => {
+        throw new NetworkConnectionError();
+      }).toThrow(NetworkConnectionError);
+
+      expect(() => {
+        throw new NetworkConnectionError();
+      }).toThrow('Network connection error');
+    });
+
+    it('should always have the same message regardless of parameters', () => {
+      const error1 = new NetworkConnectionError();
+      const error2 = new NetworkConnectionError();
+
+      expect(error1.message).toBe(error2.message);
+      expect(error1.name).toBe(error2.name);
+      expect(error1.message).toBe('Network connection error');
+    });
+  });
+
+  describe('TimeoutError', () => {
+    it('should create error with correct message and name', () => {
+      const message = 'Request timeout after 30 seconds';
+      const error = new TimeoutError(message);
+
+      expect(error).toBeInstanceOf(Error);
+      expect(error).toBeInstanceOf(TimeoutError);
+      expect(error.message).toBe(message);
+      expect(error.name).toBe('TimeoutError');
+    });
+
+    it('should preserve stack trace', () => {
+      const error = new TimeoutError('Test timeout');
+      expect(error.stack).toBeDefined();
+      expect(typeof error.stack).toBe('string');
+    });
+
+    it('should be throwable and catchable', () => {
+      const message = 'Custom timeout message';
+
+      expect(() => {
+        throw new TimeoutError(message);
+      }).toThrow(TimeoutError);
+
+      expect(() => {
+        throw new TimeoutError(message);
+      }).toThrow(message);
+    });
+
+    it('should handle empty message', () => {
+      const error = new TimeoutError('');
+      expect(error.message).toBe('');
+      expect(error.name).toBe('TimeoutError');
+    });
+
+    it('should handle numeric timeout values in message', () => {
+      const timeoutMessage = 'Request timed out after 5000ms';
+      const error = new TimeoutError(timeoutMessage);
+      expect(error.message).toBe(timeoutMessage);
+      expect(error.name).toBe('TimeoutError');
+    });
+  });
+
+  describe('error inheritance', () => {
+    it('should all extend from base Error class', () => {
+      const pageError = new PageNotFoundError('404');
+      const networkError = new NetworkConnectionError();
+      const timeoutError = new TimeoutError('timeout');
+
+      expect(pageError).toBeInstanceOf(Error);
+      expect(networkError).toBeInstanceOf(Error);
+      expect(timeoutError).toBeInstanceOf(Error);
+    });
+
+    it('should be distinguishable by instanceof', () => {
+      const pageError = new PageNotFoundError('404');
+      const networkError = new NetworkConnectionError();
+      const timeoutError = new TimeoutError('timeout');
+
+      expect(pageError).toBeInstanceOf(PageNotFoundError);
+      expect(pageError).not.toBeInstanceOf(NetworkConnectionError);
+      expect(pageError).not.toBeInstanceOf(TimeoutError);
+
+      expect(networkError).toBeInstanceOf(NetworkConnectionError);
+      expect(networkError).not.toBeInstanceOf(PageNotFoundError);
+      expect(networkError).not.toBeInstanceOf(TimeoutError);
+
+      expect(timeoutError).toBeInstanceOf(TimeoutError);
+      expect(timeoutError).not.toBeInstanceOf(PageNotFoundError);
+      expect(timeoutError).not.toBeInstanceOf(NetworkConnectionError);
+    });
+
+    it('should be distinguishable by name property', () => {
+      const pageError = new PageNotFoundError('404');
+      const networkError = new NetworkConnectionError();
+      const timeoutError = new TimeoutError('timeout');
+
+      expect(pageError.name).toBe('PageNotFoundError');
+      expect(networkError.name).toBe('NetworkConnectionError');
+      expect(timeoutError.name).toBe('TimeoutError');
+
+      // Names should be unique
+      const names = [pageError.name, networkError.name, timeoutError.name];
+      const uniqueNames = [...new Set(names)];
+      expect(uniqueNames).toHaveLength(names.length);
+    });
+  });
+
+  describe('error catching scenarios', () => {
+    it('should allow catching specific error types', () => {
+      const testErrors = [
+        new PageNotFoundError('404 error'),
+        new NetworkConnectionError(),
+        new TimeoutError('timeout error'),
+      ];
+
+      testErrors.forEach((error) => {
+        try {
+          throw error;
+        } catch (e) {
+          if (e instanceof PageNotFoundError) {
+            expect(e.name).toBe('PageNotFoundError');
+            expect(e.message).toBe('404 error');
+          } else if (e instanceof NetworkConnectionError) {
+            expect(e.name).toBe('NetworkConnectionError');
+            expect(e.message).toBe('Network connection error');
+          } else if (e instanceof TimeoutError) {
+            expect(e.name).toBe('TimeoutError');
+            expect(e.message).toBe('timeout error');
+          } else {
+            throw new Error('Unexpected error type');
+          }
+        }
+      });
+    });
+
+    it('should allow catching by error name', () => {
+      const testCases = [
+        { error: new PageNotFoundError('test'), expectedName: 'PageNotFoundError' },
+        { error: new NetworkConnectionError(), expectedName: 'NetworkConnectionError' },
+        { error: new TimeoutError('test'), expectedName: 'TimeoutError' },
+      ];
+
+      testCases.forEach(({ error, expectedName }) => {
+        try {
+          throw error;
+        } catch (e) {
+          expect((e as Error).name).toBe(expectedName);
+        }
+      });
+    });
+  });
+});

package/packages/web-crawler/vitest.config.mts

@@ -2,6 +2,9 @@ import { defineConfig } from 'vitest/config';
 
 export default defineConfig({
   test: {
+    coverage: {
+      reporter: ['text', 'json', 'lcov', 'text-summary'],
+    },
     environment: 'node',
   },
 });