@lobehub/chat 1.124.0 → 1.124.2

package/packages/web-crawler/src/crawImpl/__tests__/firecrawl.test.ts

@@ -0,0 +1,284 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../../utils/errorType';
+import { firecrawl } from '../firecrawl';
+
+// Mock dependencies
+vi.mock('../../utils/withTimeout', () => ({
+  DEFAULT_TIMEOUT: 30000,
+  withTimeout: vi.fn(),
+}));
+
+describe('firecrawl crawler', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    delete process.env.FIRECRAWL_API_KEY;
+    delete process.env.FIRECRAWL_URL;
+  });
+
+  it('should successfully crawl content with API key', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        success: true,
+        data: {
+          markdown:
+            'This is a test markdown content with enough length to pass validation. '.repeat(3),
+          metadata: {
+            title: 'Test Article',
+            description: 'Test description',
+            sourceURL: 'https://example.com',
+            statusCode: 200,
+            language: 'en',
+            keywords: 'test',
+            robots: 'index',
+          },
+        },
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await firecrawl('https://example.com', { filterOptions: {} });
+
+    expect(result).toEqual({
+      content: 'This is a test markdown content with enough length to pass validation. '.repeat(3),
+      contentType: 'text',
+      description: 'Test description',
+      length: 'This is a test markdown content with enough length to pass validation. '.repeat(3)
+        .length,
+      siteName: 'example.com',
+      title: 'Test Article',
+      url: 'https://example.com',
+    });
+
+    expect(withTimeout).toHaveBeenCalledWith(expect.any(Promise), 30000);
+  });
+
+  it('should handle missing API key', async () => {
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        success: true,
+        data: {
+          markdown: 'Test content with sufficient length. '.repeat(5),
+          metadata: {
+            title: 'Test',
+            description: 'Test',
+            sourceURL: 'https://example.com',
+            statusCode: 200,
+            language: 'en',
+            keywords: 'test',
+            robots: 'index',
+          },
+        },
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await firecrawl('https://example.com', { filterOptions: {} });
+
+    expect(withTimeout).toHaveBeenCalledWith(expect.any(Promise), 30000);
+  });
+
+  it('should return undefined for short content', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        success: true,
+        data: {
+          markdown: 'Short', // Content too short
+          metadata: {
+            title: 'Test',
+            description: 'Test',
+            sourceURL: 'https://example.com',
+            statusCode: 200,
+            language: 'en',
+            keywords: 'test',
+            robots: 'index',
+          },
+        },
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await firecrawl('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should return undefined when markdown is missing', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        success: true,
+        data: {
+          // markdown is missing
+          metadata: {
+            title: 'Test',
+            description: 'Test',
+            sourceURL: 'https://example.com',
+            statusCode: 200,
+            language: 'en',
+            keywords: 'test',
+            robots: 'index',
+          },
+        },
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await firecrawl('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should throw PageNotFoundError for 404 status', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: false,
+      status: 404,
+      statusText: 'Not Found',
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await expect(firecrawl('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      PageNotFoundError,
+    );
+  });
+
+  it('should throw error for other HTTP errors', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: false,
+      status: 500,
+      statusText: 'Internal Server Error',
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await expect(firecrawl('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      'Firecrawl request failed with status 500: Internal Server Error',
+    );
+  });
+
+  it('should throw NetworkConnectionError for fetch failures', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(new Error('fetch failed'));
+
+    await expect(firecrawl('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      NetworkConnectionError,
+    );
+  });
+
+  it('should throw TimeoutError when request times out', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const timeoutError = new TimeoutError('Request timeout');
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(timeoutError);
+
+    await expect(firecrawl('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      TimeoutError,
+    );
+  });
+
+  it('should rethrow unknown errors', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const unknownError = new Error('Unknown error');
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(unknownError);
+
+    await expect(firecrawl('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      'Unknown error',
+    );
+  });
+
+  it('should return undefined when JSON parsing fails', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockRejectedValue(new Error('Invalid JSON')),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    const result = await firecrawl('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+    expect(consoleSpy).toHaveBeenCalled();
+
+    consoleSpy.mockRestore();
+  });
+
+  it('should handle metadata with all optional fields', async () => {
+    process.env.FIRECRAWL_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        success: true,
+        data: {
+          markdown: 'Complete test content with all metadata fields provided. '.repeat(3),
+          metadata: {
+            title: 'Complete Test Article',
+            description: 'Complete test description',
+            keywords: 'test,complete,article',
+            language: 'en',
+            ogDescription: 'OG description',
+            ogImage: 'https://example.com/image.jpg',
+            ogLocaleAlternate: ['en-US', 'fr-FR'],
+            ogSiteName: 'Example Site',
+            ogTitle: 'OG Title',
+            ogUrl: 'https://example.com/og',
+            robots: 'index,follow',
+            statusCode: 200,
+            sourceURL: 'https://example.com',
+          },
+        },
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await firecrawl('https://example.com', { filterOptions: {} });
+
+    expect(result).toEqual({
+      content: 'Complete test content with all metadata fields provided. '.repeat(3),
+      contentType: 'text',
+      description: 'Complete test description',
+      length: 'Complete test content with all metadata fields provided. '.repeat(3).length,
+      siteName: 'example.com',
+      title: 'Complete Test Article',
+      url: 'https://example.com',
+    });
+  });
+});

package/packages/web-crawler/src/crawImpl/__tests__/naive.test.ts

@@ -0,0 +1,234 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../../utils/errorType';
+import { naive } from '../naive';
+
+// Mock dependencies
+vi.mock('../../utils/htmlToMarkdown', () => ({
+  htmlToMarkdown: vi.fn(),
+}));
+
+vi.mock('../../utils/withTimeout', () => ({
+  DEFAULT_TIMEOUT: 30000,
+  withTimeout: vi.fn(),
+}));
+
+// Mock fetch globally
+const mockFetch = vi.fn();
+global.fetch = mockFetch;
+
+describe('naive crawler', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should return undefined for normal pages (due to cloudflare logic)', async () => {
+    const mockResponse = {
+      status: 200,
+      headers: new Map([['content-type', 'text/html']]),
+      text: vi.fn().mockResolvedValue('<html><body>Test content</body></html>'),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const { htmlToMarkdown } = await import('../../utils/htmlToMarkdown');
+    vi.mocked(htmlToMarkdown).mockReturnValue({
+      content: 'Test content'.padEnd(101, ' '), // Ensure length > 100
+      title: 'Normal Page Title', // Not "Just a moment..." so it returns undefined
+      description: 'Test description',
+      siteName: 'Test Site',
+      length: 101,
+    });
+
+    const result = await naive('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should successfully crawl JSON content', async () => {
+    const mockJsonData = { message: 'Hello world', data: [1, 2, 3] };
+    const mockResponse = {
+      status: 200,
+      headers: new Map([['content-type', 'application/json']]),
+      clone: () => ({
+        json: vi.fn().mockResolvedValue(mockJsonData),
+      }),
+      text: vi.fn().mockResolvedValue(JSON.stringify(mockJsonData)),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await naive('https://api.example.com/data', { filterOptions: {} });
+
+    expect(result).toEqual({
+      content: JSON.stringify(mockJsonData, null, 2),
+      contentType: 'json',
+      length: JSON.stringify(mockJsonData, null, 2).length,
+      url: 'https://api.example.com/data',
+    });
+  });
+
+  it('should handle malformed JSON by falling back to text', async () => {
+    const mockText = '{"invalid": json}';
+    const mockResponse = {
+      status: 200,
+      headers: new Map([['content-type', 'application/json']]),
+      clone: () => ({
+        json: vi.fn().mockRejectedValue(new Error('Invalid JSON')),
+      }),
+      text: vi.fn().mockResolvedValue(mockText),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await naive('https://api.example.com/data', { filterOptions: {} });
+
+    expect(result).toEqual({
+      content: mockText,
+      contentType: 'json',
+      length: mockText.length,
+      url: 'https://api.example.com/data',
+    });
+  });
+
+  it('should return undefined for short content', async () => {
+    const mockResponse = {
+      status: 200,
+      headers: new Map([['content-type', 'text/html']]),
+      text: vi.fn().mockResolvedValue('<html><body>Short</body></html>'),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const { htmlToMarkdown } = await import('../../utils/htmlToMarkdown');
+    vi.mocked(htmlToMarkdown).mockReturnValue({
+      content: 'Short', // Length < 100
+      title: 'Test Page',
+      length: 5,
+    });
+
+    const result = await naive('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should return content when NOT blocked by Cloudflare', async () => {
+    const mockResponse = {
+      status: 200,
+      headers: new Map([['content-type', 'text/html']]),
+      text: vi.fn().mockResolvedValue('<html><body>Normal content</body></html>'),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const { htmlToMarkdown } = await import('../../utils/htmlToMarkdown');
+    vi.mocked(htmlToMarkdown).mockReturnValue({
+      content: 'Test content'.padEnd(101, ' '),
+      title: 'Just a moment...', // Cloudflare blocking page - this will cause return
+      description: 'Test description',
+      siteName: 'Test Site',
+      length: 101,
+    });
+
+    const result = await naive('https://example.com', { filterOptions: {} });
+
+    expect(result).toEqual({
+      content: 'Test content'.padEnd(101, ' '),
+      contentType: 'text',
+      description: 'Test description',
+      length: 101,
+      siteName: 'Test Site',
+      title: 'Just a moment...',
+      url: 'https://example.com',
+    });
+  });
+
+  it('should throw PageNotFoundError for 404 status', async () => {
+    const mockResponse = {
+      status: 404,
+      statusText: 'Not Found',
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await expect(naive('https://example.com/notfound', { filterOptions: {} })).rejects.toThrow(
+      PageNotFoundError,
+    );
+  });
+
+  it('should throw NetworkConnectionError for fetch failures', async () => {
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(new Error('fetch failed'));
+
+    await expect(naive('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      NetworkConnectionError,
+    );
+  });
+
+  it('should throw TimeoutError when request times out', async () => {
+    const timeoutError = new TimeoutError('Request timeout');
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(timeoutError);
+
+    await expect(naive('https://example.com', { filterOptions: {} })).rejects.toThrow(TimeoutError);
+  });
+
+  it('should rethrow unknown errors', async () => {
+    const unknownError = new Error('Unknown error');
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(unknownError);
+
+    await expect(naive('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      'Unknown error',
+    );
+  });
+
+  it('should return undefined when HTML processing fails', async () => {
+    const mockResponse = {
+      status: 200,
+      headers: new Map([['content-type', 'text/html']]),
+      text: vi.fn().mockRejectedValue(new Error('Failed to read text')),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await naive('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should pass filter options to htmlToMarkdown', async () => {
+    const mockResponse = {
+      status: 200,
+      headers: new Map([['content-type', 'text/html']]),
+      text: vi.fn().mockResolvedValue('<html><body>Test content</body></html>'),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const { htmlToMarkdown } = await import('../../utils/htmlToMarkdown');
+    vi.mocked(htmlToMarkdown).mockReturnValue({
+      content: 'Test content'.padEnd(101, ' '),
+      title: 'Test Page',
+      length: 101,
+    });
+
+    const filterOptions = { enableReadability: true, pureText: false };
+    await naive('https://example.com', { filterOptions });
+
+    expect(htmlToMarkdown).toHaveBeenCalledWith('<html><body>Test content</body></html>', {
+      filterOptions,
+      url: 'https://example.com',
+    });
+  });
+});