@lobehub/chat 1.124.1 → 1.124.3

package/packages/utils/src/electron/desktopRemoteRPCFetch.ts

@@ -1,7 +1,7 @@
+import { isDesktop } from '@lobechat/const';
 import { ProxyTRPCRequestParams, dispatch, streamInvoke } from '@lobechat/electron-client-ipc';
 import debug from 'debug';
 
-import { isDesktop } from '@/const/version';
 import { getElectronStoreState } from '@/store/electron';
 import { electronSyncSelectors } from '@/store/electron/selectors';
 import { getRequestBody, headersToRecord } from '@/utils/fetch';

package/packages/utils/src/fetch/fetchSSE.ts

@@ -1,20 +1,19 @@
+import { LOBE_CHAT_OBSERVATION_ID, LOBE_CHAT_TRACE_ID, MESSAGE_CANCEL_FLAT } from '@lobechat/const';
 import { parseToolCalls } from '@lobechat/model-runtime';
-import { ChatErrorType } from '@lobechat/types';
-
-import { MESSAGE_CANCEL_FLAT } from '@/const/message';
-import { LOBE_CHAT_OBSERVATION_ID, LOBE_CHAT_TRACE_ID } from '@/const/trace';
-import { ResponseAnimation, ResponseAnimationStyle } from '@lobechat/types';
 import {
+  ChatErrorType,
+  ChatImageChunk,
   ChatMessageError,
+  GroundingSearch,
   MessageToolCall,
   MessageToolCallChunk,
   MessageToolCallSchema,
   ModelReasoning,
   ModelSpeed,
   ModelTokensUsage,
-} from '@/types/message';
-import { ChatImageChunk } from '@/types/message/image';
-import { GroundingSearch } from '@/types/search';
+  ResponseAnimation,
+  ResponseAnimationStyle,
+} from '@lobechat/types';
 
 import { nanoid } from '../uuid';
 import { fetchEventSource } from './fetchEventSource';

package/packages/utils/src/fetch/parseError.ts

@@ -1,8 +1,6 @@
-import { ErrorResponse, ErrorType } from '@lobechat/types';
+import { ChatMessageError, ErrorResponse, ErrorType } from '@lobechat/types';
 import { t } from 'i18next';
 
-import { ChatMessageError } from '@/types/message';
-
 export const getMessageError = async (response: Response) => {
   let chatMessageError: ChatMessageError;
 

package/packages/utils/src/format.test.ts

@@ -1,8 +1,7 @@
+import { USD_TO_CNY } from '@lobechat/const';
 import dayjs from 'dayjs';
 import { describe, expect, it } from 'vitest';
 
-import { USD_TO_CNY } from '@/const/currency';
-
 import {
   formatDate,
   formatIntergerNumber,

package/packages/utils/src/index.ts

@@ -7,4 +7,5 @@ export * from './object';
 export * from './parseModels';
 export * from './pricing';
 export * from './safeParseJSON';
+export * from './sleep';
 export * from './uuid';

package/packages/utils/src/toolManifest.ts

@@ -1,9 +1,8 @@
+import { ChatCompletionTool , OpenAIPluginManifest } from '@lobechat/types';
 import { LobeChatPluginManifest, pluginManifestSchema } from '@lobehub/chat-plugin-sdk';
 import { uniqBy } from 'lodash-es';
 
 import { API_ENDPOINTS } from '@/services/_url';
-import { ChatCompletionTool } from '@/types/openai/chat';
-import { OpenAIPluginManifest } from '@/types/openai/plugin';
 import { genToolCallingName } from '@/utils/toolCall';
 
 const fetchJSON = async <T = any>(url: string, proxy = false): Promise<T> => {

package/packages/utils/src/trace.ts

@@ -1,4 +1,4 @@
-import { LOBE_CHAT_TRACE_HEADER, LOBE_CHAT_TRACE_ID, TracePayload } from '@/const/trace';
+import { LOBE_CHAT_TRACE_HEADER, LOBE_CHAT_TRACE_ID, TracePayload } from '@lobechat/const';
 
 export const getTracePayload = (req: Request): TracePayload | undefined => {
   const header = req.headers.get(LOBE_CHAT_TRACE_HEADER);

package/packages/utils/vitest.config.mts

@@ -5,13 +5,12 @@ export default defineConfig({
   test: {
     alias: {
       /* eslint-disable sort-keys-fix/sort-keys-fix */
-      '@/types': resolve(__dirname, '../types/src'),
       '@/const': resolve(__dirname, '../const/src'),
-      '@/libs/model-runtime': resolve(__dirname, '../model-runtime/src'),
       '@': resolve(__dirname, '../../src'),
       /* eslint-enable */
     },
     coverage: {
+      all: false,
       reporter: ['text', 'json', 'lcov', 'text-summary'],
     },
     environment: 'happy-dom',

package/packages/web-crawler/src/__tests__/urlRules.test.ts

@@ -0,0 +1,275 @@
+import { describe, expect, it } from 'vitest';
+
+import { crawUrlRules } from '../urlRules';
+
+describe('urlRules', () => {
+  it('should export an array of CrawlUrlRule objects', () => {
+    expect(Array.isArray(crawUrlRules)).toBe(true);
+    expect(crawUrlRules.length).toBeGreaterThan(0);
+  });
+
+  it('should have valid rule structure for each rule', () => {
+    crawUrlRules.forEach((rule, index) => {
+      expect(rule).toHaveProperty('urlPattern');
+      expect(typeof rule.urlPattern).toBe('string');
+      expect(rule.urlPattern.length).toBeGreaterThan(0);
+
+      if (rule.impls) {
+        expect(Array.isArray(rule.impls)).toBe(true);
+        expect(rule.impls.length).toBeGreaterThan(0);
+      }
+
+      if (rule.filterOptions) {
+        expect(typeof rule.filterOptions).toBe('object');
+      }
+
+      if (rule.urlTransform) {
+        expect(typeof rule.urlTransform).toBe('string');
+        expect(rule.urlTransform.length).toBeGreaterThan(0);
+      }
+    });
+  });
+
+  describe('specific URL patterns', () => {
+    it('should match WeChat Sogou links', () => {
+      const wechatRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://weixin.sogou.com/link(.*)',
+      );
+
+      expect(wechatRule).toBeDefined();
+      expect(wechatRule?.impls).toEqual(['search1api']);
+    });
+
+    it('should match Sogou links', () => {
+      const sogouRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://sogou.com/link(.*)',
+      );
+
+      expect(sogouRule).toBeDefined();
+      expect(sogouRule?.impls).toEqual(['search1api']);
+    });
+
+    it('should match YouTube links', () => {
+      const youtubeRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://www.youtube.com/watch(.*)',
+      );
+
+      expect(youtubeRule).toBeDefined();
+      expect(youtubeRule?.impls).toEqual(['search1api']);
+    });
+
+    it('should match Reddit links', () => {
+      const redditRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://www.reddit.com/r/(.*)/comments/(.*)',
+      );
+
+      expect(redditRule).toBeDefined();
+      expect(redditRule?.impls).toEqual(['search1api']);
+    });
+
+    it('should match WeChat public account links', () => {
+      const wechatPublicRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://mp.weixin.qq.com(.*)',
+      );
+
+      expect(wechatPublicRule).toBeDefined();
+      expect(wechatPublicRule?.impls).toEqual(['search1api', 'jina']);
+    });
+
+    it('should match GitHub blob links with URL transformation', () => {
+      const githubBlobRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://github.com/([^/]+)/([^/]+)/blob/([^/]+)/(.*)',
+      );
+
+      expect(githubBlobRule).toBeDefined();
+      expect(githubBlobRule?.impls).toEqual(['naive', 'jina']);
+      expect(githubBlobRule?.urlTransform).toBe('https://github.com/$1/$2/raw/refs/heads/$3/$4');
+      expect(githubBlobRule?.filterOptions?.enableReadability).toBe(false);
+    });
+
+    it('should match GitHub discussion links', () => {
+      const githubDiscussionRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://github.com/(.*)/discussions/(.*)',
+      );
+
+      expect(githubDiscussionRule).toBeDefined();
+      expect(githubDiscussionRule?.impls).toEqual(['naive', 'jina']);
+      expect(githubDiscussionRule?.filterOptions?.enableReadability).toBe(false);
+    });
+
+    it('should match PDF files', () => {
+      const pdfRule = crawUrlRules.find((rule) => rule.urlPattern === 'https://(.*).pdf');
+
+      expect(pdfRule).toBeDefined();
+      expect(pdfRule?.impls).toEqual(['jina']);
+    });
+
+    it('should match arXiv PDF links', () => {
+      const arxivRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://arxiv.org/pdf/(.*)',
+      );
+
+      expect(arxivRule).toBeDefined();
+      expect(arxivRule?.impls).toEqual(['jina']);
+    });
+
+    it('should match Zhihu Zhuanlan links', () => {
+      const zhihuZhuanlanRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://zhuanlan.zhihu.com(.*)',
+      );
+
+      expect(zhihuZhuanlanRule).toBeDefined();
+      expect(zhihuZhuanlanRule?.impls).toEqual(['jina']);
+    });
+
+    it('should match Zhihu links', () => {
+      const zhihuRule = crawUrlRules.find((rule) => rule.urlPattern === 'https://zhihu.com(.*)');
+
+      expect(zhihuRule).toBeDefined();
+      expect(zhihuRule?.impls).toEqual(['jina']);
+    });
+
+    it('should match Medium links with URL transformation', () => {
+      const mediumRule = crawUrlRules.find((rule) => rule.urlPattern === 'https://medium.com/(.*)');
+
+      expect(mediumRule).toBeDefined();
+      expect(mediumRule?.urlTransform).toBe('https://scribe.rip/$1');
+    });
+
+    it('should match Twitter/X links', () => {
+      const twitterRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://(twitter.com|x.com)/(.*)',
+      );
+
+      expect(twitterRule).toBeDefined();
+      expect(twitterRule?.impls).toEqual(['jina', 'browserless']);
+      expect(twitterRule?.filterOptions?.enableReadability).toBe(false);
+    });
+
+    it('should match sports data website with specific filter options', () => {
+      const sportsRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://www.qiumiwu.com/standings/(.*)',
+      );
+
+      expect(sportsRule).toBeDefined();
+      expect(sportsRule?.impls).toEqual(['naive']);
+      expect(sportsRule?.filterOptions?.enableReadability).toBe(false);
+      expect(sportsRule?.filterOptions?.pureText).toBe(true);
+    });
+
+    it('should match Mozilla Developer docs', () => {
+      const mozillaRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://developer.mozilla.org(.*)',
+      );
+
+      expect(mozillaRule).toBeDefined();
+      expect(mozillaRule?.impls).toEqual(['jina']);
+    });
+
+    it('should match CVPR links', () => {
+      const cvprRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://cvpr.thecvf.com(.*)',
+      );
+
+      expect(cvprRule).toBeDefined();
+      expect(cvprRule?.impls).toEqual(['jina']);
+    });
+
+    it('should match Feishu links', () => {
+      const feishuRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://(.*).feishu.cn/(.*)',
+      );
+
+      expect(feishuRule).toBeDefined();
+      expect(feishuRule?.impls).toEqual(['jina']);
+    });
+
+    it('should match Xiaohongshu (Little Red Book) links', () => {
+      const xiaohongshuRule = crawUrlRules.find(
+        (rule) => rule.urlPattern === 'https://(.*).xiaohongshu.com/(.*)',
+      );
+
+      expect(xiaohongshuRule).toBeDefined();
+      expect(xiaohongshuRule?.impls).toEqual(['search1api', 'jina']);
+    });
+  });
+
+  describe('URL pattern validation', () => {
+    it('should have valid regex patterns that can be compiled', () => {
+      crawUrlRules.forEach((rule, index) => {
+        expect(() => {
+          new RegExp(rule.urlPattern);
+        }).not.toThrow(`Rule at index ${index} should have valid regex pattern`);
+      });
+    });
+  });
+
+  describe('impl validation', () => {
+    const validImpls = ['naive', 'jina', 'browserless', 'search1api', 'exa', 'firecrawl', 'tavily'];
+
+    it('should only use valid crawler implementations', () => {
+      crawUrlRules.forEach((rule, index) => {
+        if (rule.impls) {
+          rule.impls.forEach((impl) => {
+            expect(validImpls).toContain(impl);
+          });
+        }
+      });
+    });
+  });
+
+  describe('filter options validation', () => {
+    it('should have valid filter options structure', () => {
+      crawUrlRules.forEach((rule, index) => {
+        if (rule.filterOptions) {
+          const { filterOptions } = rule;
+
+          if ('enableReadability' in filterOptions) {
+            expect(typeof filterOptions.enableReadability).toBe('boolean');
+          }
+
+          if ('pureText' in filterOptions) {
+            expect(typeof filterOptions.pureText).toBe('boolean');
+          }
+        }
+      });
+    });
+  });
+
+  describe('URL transformation validation', () => {
+    it('should have valid URL transformation templates', () => {
+      crawUrlRules.forEach((rule, index) => {
+        if (rule.urlTransform) {
+          // Check if URL transform contains placeholder groups
+          expect(rule.urlTransform).toMatch(/\$\d+/);
+
+          // Check if it's a valid URL format
+          expect(rule.urlTransform).toMatch(/^https?:\/\//);
+        }
+      });
+    });
+  });
+
+  describe('rule completeness', () => {
+    it('should cover major social media platforms', () => {
+      const patterns = crawUrlRules.map((rule) => rule.urlPattern);
+
+      // Check for major platforms
+      expect(patterns.some((p) => p.includes('youtube.com'))).toBe(true);
+      expect(patterns.some((p) => p.includes('reddit.com'))).toBe(true);
+      expect(patterns.some((p) => p.includes('twitter.com') || p.includes('x.com'))).toBe(true);
+      expect(patterns.some((p) => p.includes('medium.com'))).toBe(true);
+      expect(patterns.some((p) => p.includes('github.com'))).toBe(true);
+    });
+
+    it('should cover Chinese platforms', () => {
+      const patterns = crawUrlRules.map((rule) => rule.urlPattern);
+
+      // Check for Chinese platforms
+      expect(patterns.some((p) => p.includes('weixin.sogou.com'))).toBe(true);
+      expect(patterns.some((p) => p.includes('zhihu.com'))).toBe(true);
+      expect(patterns.some((p) => p.includes('xiaohongshu.com'))).toBe(true);
+      expect(patterns.some((p) => p.includes('feishu.cn'))).toBe(true);
+    });
+  });
+});

package/packages/web-crawler/src/crawImpl/__tests__/exa.test.ts

@@ -0,0 +1,269 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../../utils/errorType';
+import { exa } from '../exa';
+
+// Mock dependencies
+vi.mock('../../utils/withTimeout', () => ({
+  DEFAULT_TIMEOUT: 30000,
+  withTimeout: vi.fn(),
+}));
+
+describe('exa crawler', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    delete process.env.EXA_API_KEY;
+  });
+
+  it('should successfully crawl content with API key', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        requestId: 'test-request-id',
+        results: [
+          {
+            id: 'test-id',
+            title: 'Test Article',
+            url: 'https://example.com',
+            text: 'This is a test article with enough content to pass the length check. '.repeat(3),
+            author: 'Test Author',
+            publishedDate: '2023-01-01',
+            summary: 'Test summary',
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await exa('https://example.com', { filterOptions: {} });
+
+    expect(result).toEqual({
+      content: 'This is a test article with enough content to pass the length check. '.repeat(3),
+      contentType: 'text',
+      length: 'This is a test article with enough content to pass the length check. '.repeat(3)
+        .length,
+      siteName: 'example.com',
+      title: 'Test Article',
+      url: 'https://example.com',
+    });
+
+    expect(withTimeout).toHaveBeenCalledWith(expect.any(Promise), 30000);
+  });
+
+  it('should handle missing API key', async () => {
+    // API key is undefined
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        results: [
+          {
+            title: 'Test Article',
+            url: 'https://example.com',
+            text: 'Test content with sufficient length. '.repeat(5),
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await exa('https://example.com', { filterOptions: {} });
+
+    // Check that fetch was called with empty API key header
+    expect(withTimeout).toHaveBeenCalledWith(expect.any(Promise), 30000);
+  });
+
+  it('should return undefined when no results are returned', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        requestId: 'test-request-id',
+        results: [],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    const result = await exa('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+    expect(consoleSpy).toHaveBeenCalledWith(
+      'Exa API returned no results for URL:',
+      'https://example.com',
+    );
+
+    consoleSpy.mockRestore();
+  });
+
+  it('should return undefined for short content', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        results: [
+          {
+            title: 'Test Article',
+            url: 'https://example.com',
+            text: 'Short', // Content too short
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await exa('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+  });
+
+  it('should throw PageNotFoundError for 404 status', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: false,
+      status: 404,
+      statusText: 'Not Found',
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await expect(exa('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      PageNotFoundError,
+    );
+  });
+
+  it('should throw error for other HTTP errors', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: false,
+      status: 500,
+      statusText: 'Internal Server Error',
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    await expect(exa('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      'Exa request failed with status 500: Internal Server Error',
+    );
+  });
+
+  it('should throw NetworkConnectionError for fetch failures', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(new Error('fetch failed'));
+
+    await expect(exa('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      NetworkConnectionError,
+    );
+  });
+
+  it('should throw TimeoutError when request times out', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const timeoutError = new TimeoutError('Request timeout');
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(timeoutError);
+
+    await expect(exa('https://example.com', { filterOptions: {} })).rejects.toThrow(TimeoutError);
+  });
+
+  it('should rethrow unknown errors', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const unknownError = new Error('Unknown error');
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockRejectedValue(unknownError);
+
+    await expect(exa('https://example.com', { filterOptions: {} })).rejects.toThrow(
+      'Unknown error',
+    );
+  });
+
+  it('should return undefined when JSON parsing fails', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockRejectedValue(new Error('Invalid JSON')),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    const result = await exa('https://example.com', { filterOptions: {} });
+
+    expect(result).toBeUndefined();
+    expect(consoleSpy).toHaveBeenCalled();
+
+    consoleSpy.mockRestore();
+  });
+
+  it('should use result URL when available', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        results: [
+          {
+            title: 'Test Article',
+            url: 'https://redirected.example.com',
+            text: 'Test content with sufficient length. '.repeat(5),
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await exa('https://example.com', { filterOptions: {} });
+
+    expect(result?.url).toBe('https://redirected.example.com');
+  });
+
+  it('should fallback to original URL when result URL is missing', async () => {
+    process.env.EXA_API_KEY = 'test-api-key';
+
+    const mockResponse = {
+      ok: true,
+      json: vi.fn().mockResolvedValue({
+        results: [
+          {
+            title: 'Test Article',
+            text: 'Test content with sufficient length. '.repeat(5),
+            // url is missing
+          },
+        ],
+      }),
+    };
+
+    const { withTimeout } = await import('../../utils/withTimeout');
+    vi.mocked(withTimeout).mockResolvedValue(mockResponse as any);
+
+    const result = await exa('https://example.com', { filterOptions: {} });
+
+    expect(result?.url).toBe('https://example.com');
+  });
+});