@lobehub/chat 1.80.5 → 1.81.0

package/packages/web-crawler/src/crawImpl/__tests__/search1api.test.ts

@@ -1,14 +1,14 @@
 import { describe, expect, it, vi } from 'vitest';
 
-import * as withTimeoutModule from '../../utils/withTimeout';
 import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../../utils/errorType';
+import * as withTimeoutModule from '../../utils/withTimeout';
 import { search1api } from '../search1api';
 
 describe('search1api crawler', () => {
   // Mock fetch function
   const mockFetch = vi.fn();
   global.fetch = mockFetch;
-  
+
   // Original env
   let originalEnv: NodeJS.ProcessEnv;
 
@@ -16,7 +16,7 @@ describe('search1api crawler', () => {
     vi.resetAllMocks();
     originalEnv = { ...process.env };
     process.env.SEARCH1API_API_KEY = 'test-api-key';
-    
+
     // Mock withTimeout to directly return the promise
     vi.spyOn(withTimeoutModule, 'withTimeout').mockImplementation((promise) => promise);
   });
@@ -25,17 +25,9 @@ describe('search1api crawler', () => {
     process.env = originalEnv;
   });
 
-  it('should throw error when API key is not set', async () => {
-    delete process.env.SEARCH1API_API_KEY;
-    
-    await expect(search1api('https://example.com', { filterOptions: {} })).rejects.toThrow(
-      'SEARCH1API_API_KEY environment variable is not set',
-    );
-  });
-
   it('should throw NetworkConnectionError when fetch fails', async () => {
     mockFetch.mockRejectedValue(new Error('fetch failed'));
-    
+
     await expect(search1api('https://example.com', { filterOptions: {} })).rejects.toThrow(
       NetworkConnectionError,
     );
@@ -44,12 +36,12 @@ describe('search1api crawler', () => {
   it('should throw TimeoutError when request times out', async () => {
     // Restore original withTimeout implementation for this test
     vi.spyOn(withTimeoutModule, 'withTimeout').mockRestore();
-    
+
     // Mock withTimeout to throw TimeoutError
     vi.spyOn(withTimeoutModule, 'withTimeout').mockImplementation(() => {
       throw new TimeoutError('Request timeout after 10000ms');
     });
-    
+
     await expect(search1api('https://example.com', { filterOptions: {} })).rejects.toThrow(
       TimeoutError,
     );
@@ -61,7 +53,7 @@ describe('search1api crawler', () => {
       status: 404,
       statusText: 'Not Found',
     });
-    
+
     await expect(search1api('https://example.com', { filterOptions: {} })).rejects.toThrow(
       PageNotFoundError,
     );
@@ -73,7 +65,7 @@ describe('search1api crawler', () => {
       status: 500,
       statusText: 'Internal Server Error',
     });
-    
+
     await expect(search1api('https://example.com', { filterOptions: {} })).rejects.toThrow(
       'Search1API request failed with status 500: Internal Server Error',
     );
@@ -82,37 +74,39 @@ describe('search1api crawler', () => {
   it('should return undefined when content is too short', async () => {
     mockFetch.mockResolvedValue({
       ok: true,
-      json: () => Promise.resolve({
-        crawlParameters: { url: 'https://example.com' },
-        results: {
-          title: 'Test Title',
-          link: 'https://example.com',
-          content: 'Short', // Less than 100 characters
-        },
-      }),
+      json: () =>
+        Promise.resolve({
+          crawlParameters: { url: 'https://example.com' },
+          results: {
+            title: 'Test Title',
+            link: 'https://example.com',
+            content: 'Short', // Less than 100 characters
+          },
+        }),
     });
-    
+
     const result = await search1api('https://example.com', { filterOptions: {} });
     expect(result).toBeUndefined();
   });
 
   it('should return crawl result on successful fetch', async () => {
     const mockContent = 'This is a test content that is longer than 100 characters. '.repeat(3);
-    
+
     mockFetch.mockResolvedValue({
       ok: true,
-      json: () => Promise.resolve({
-        crawlParameters: { url: 'https://example.com' },
-        results: {
-          title: 'Test Title',
-          link: 'https://example.com',
-          content: mockContent,
-        },
-      }),
+      json: () =>
+        Promise.resolve({
+          crawlParameters: { url: 'https://example.com' },
+          results: {
+            title: 'Test Title',
+            link: 'https://example.com',
+            content: mockContent,
+          },
+        }),
     });
-    
+
     const result = await search1api('https://example.com', { filterOptions: {} });
-    
+
     expect(mockFetch).toHaveBeenCalledWith('https://api.search1api.com/crawl', {
       method: 'POST',
       headers: {
@@ -123,7 +117,7 @@ describe('search1api crawler', () => {
         url: 'https://example.com',
       }),
     });
-    
+
     expect(result).toEqual({
       content: mockContent,
       contentType: 'text',
@@ -140,8 +134,8 @@ describe('search1api crawler', () => {
       ok: true,
       json: () => Promise.reject(new Error('Invalid JSON')),
     });
-    
+
     const result = await search1api('https://example.com', { filterOptions: {} });
     expect(result).toBeUndefined();
   });
-}); 
+});

package/packages/web-crawler/src/crawImpl/search1api.ts

@@ -17,12 +17,6 @@ export const search1api: CrawlImpl = async (url) => {
   // Get API key from environment variable
   const apiKey = process.env.SEARCH1API_CRAWL_API_KEY || process.env.SEARCH1API_API_KEY;
 
-  if (!apiKey) {
-    throw new Error(
-      'SEARCH1API_API_KEY environment variable is not set. Visit https://www.search1api.com to get free quota.',
-    );
-  }
-
   let res: Response;
 
   try {
@@ -32,7 +26,7 @@ export const search1api: CrawlImpl = async (url) => {
           url,
         }),
         headers: {
-          'Authorization': `Bearer ${apiKey}`,
+          'Authorization': !apiKey ? '' : `Bearer ${apiKey}`,
           'Content-Type': 'application/json',
         },
         method: 'POST',

package/packages/web-crawler/src/index.ts

@@ -1,2 +1,3 @@
+export type { CrawlImplType } from './crawImpl';
 export { Crawler } from './crawler';
 export * from './type';

package/packages/web-crawler/src/urlRules.ts

@@ -31,6 +31,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
     filterOptions: {
       enableReadability: false,
     },
+    impls: ['naive', 'jina'],
     urlPattern: 'https://github.com/([^/]+)/([^/]+)/blob/([^/]+)/(.*)',
     urlTransform: 'https://github.com/$1/$2/raw/refs/heads/$3/$4',
   },
@@ -38,6 +39,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
     filterOptions: {
       enableReadability: false,
     },
+    impls: ['naive', 'jina'],
     // GitHub discussion
     urlPattern: 'https://github.com/(.*)/discussions/(.*)',
   },
@@ -79,9 +81,9 @@ export const crawUrlRules: CrawlUrlRule[] = [
       enableReadability: false,
       pureText: true,
     },
+    impls: ['naive'],
     urlPattern: 'https://www.qiumiwu.com/standings/(.*)',
   },
-
   // mozilla use jina
   {
     impls: ['jina'],

package/src/config/tools.ts

@@ -5,11 +5,13 @@ export const getToolsConfig = () => {
   return createEnv({
     runtimeEnv: {
       CRAWLER_IMPLS: process.env.CRAWLER_IMPLS,
+      SEARCH_PROVIDERS: process.env.SEARCH_PROVIDERS,
       SEARXNG_URL: process.env.SEARXNG_URL,
     },
 
     server: {
       CRAWLER_IMPLS: z.string().optional(),
+      SEARCH_PROVIDERS: z.string().optional(),
       SEARXNG_URL: z.string().url().optional(),
     },
   });

package/src/features/Conversation/Messages/Assistant/Tool/Inspector/Debug.tsx

@@ -4,6 +4,7 @@ import { memo } from 'react';
 import { useTranslation } from 'react-i18next';
 
 import PluginResult from './PluginResultJSON';
+import PluginState from './PluginState';
 
 interface DebugProps {
   payload: object;
@@ -28,15 +29,20 @@ const Debug = memo<DebugProps>(({ payload, requestArgs, toolCallId }) => {
           key: 'arguments',
           label: t('debug.arguments'),
         },
+        {
+          children: <PluginResult toolCallId={toolCallId} />,
+          key: 'response',
+          label: t('debug.response'),
+        },
         {
           children: <Highlighter language={'json'}>{JSON.stringify(payload, null, 2)}</Highlighter>,
           key: 'function_call',
           label: t('debug.function_call'),
         },
         {
-          children: <PluginResult toolCallId={toolCallId} />,
-          key: 'response',
-          label: t('debug.response'),
+          children: <PluginState toolCallId={toolCallId} />,
+          key: 'pluginState',
+          label: t('debug.pluginState'),
         },
       ]}
       style={{ display: 'grid', maxWidth: 800, minWidth: 400 }}

package/src/features/Conversation/Messages/Assistant/Tool/Inspector/PluginState.tsx

@@ -0,0 +1,21 @@
+import { Highlighter } from '@lobehub/ui';
+import { memo } from 'react';
+
+import { useChatStore } from '@/store/chat';
+import { chatSelectors } from '@/store/chat/selectors';
+
+export interface FunctionMessageProps {
+  toolCallId: string;
+}
+
+const PluginState = memo<FunctionMessageProps>(({ toolCallId }) => {
+  const toolMessage = useChatStore(chatSelectors.getMessageByToolCallId(toolCallId));
+
+  return (
+    <Highlighter language={'json'} style={{ maxHeight: 200, maxWidth: 800, overflow: 'scroll' }}>
+      {JSON.stringify(toolMessage?.pluginState, null, 2)}
+    </Highlighter>
+  );
+});
+
+export default PluginState;

package/src/features/Conversation/Messages/Assistant/Tool/Render/Arguments.tsx

@@ -93,7 +93,7 @@ const ObjectDisplay = memo(({ data, shine }: ObjectDisplayProps) => {
           <div className={styles.row} key={key}>
             <span
               className={styles.key}
-              style={{ minWidth: hasMinWidth ? (isMobile ? 60 : 80) : undefined }}
+              style={{ minWidth: hasMinWidth ? (isMobile ? 60 : 140) : undefined }}
             >
               {key}
             </span>

package/src/locales/default/plugin.ts

@@ -5,6 +5,7 @@ export default {
     off: '关闭调试',
     on: '查看插件调用信息',
     payload: '插件载荷',
+    pluginState: '插件 State',
     response: '返回结果',
     tool_call: '工具调用请求',
   },

package/src/server/routers/tools/{__test__/search.test.ts → search.test.ts}

@@ -3,10 +3,10 @@ import { TRPCError } from '@trpc/server';
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { toolsEnv } from '@/config/tools';
-import { SearXNGClient } from '@/server/modules/SearXNG';
+import { SearXNGClient } from '@/server/services/search/impls/searxng/client';
 import { SEARCH_SEARXNG_NOT_CONFIG } from '@/types/tool/search';
 
-import { searchRouter } from '../search';
+import { searchRouter } from './search';
 
 // Mock JWT verification
 vi.mock('@/utils/server/jwt', () => ({
@@ -19,7 +19,7 @@ vi.mock('@lobechat/web-crawler', () => ({
   })),
 }));
 
-vi.mock('@/server/modules/SearXNG');
+vi.mock('@/server/services/search/impls/searxng/client');
 
 describe('searchRouter', () => {
   const mockContext = {
@@ -104,7 +104,18 @@ describe('searchRouter', () => {
         query: 'test query',
       });
 
-      expect(result).toEqual(mockSearchResult);
+      expect(result).toEqual({
+        costTime: 0,
+        query: 'test query',
+        results: [
+          {
+            title: 'Test Result',
+            parsedUrl: 'test.com',
+            url: 'http://test.com',
+            content: 'Test content',
+          },
+        ],
+      });
     });
 
     it('should work without specifying search engines', async () => {
@@ -128,7 +139,18 @@ describe('searchRouter', () => {
         query: 'test query',
       });
 
-      expect(result).toEqual(mockSearchResult);
+      expect(result).toEqual({
+        costTime: 0,
+        query: 'test query',
+        results: [
+          {
+            title: 'Test Result',
+            parsedUrl: 'test.com',
+            url: 'http://test.com',
+            content: 'Test content',
+          },
+        ],
+      });
     });
 
     it('should handle search errors', async () => {

package/src/server/routers/tools/search.ts

@@ -1,14 +1,9 @@
-import { Crawler } from '@lobechat/web-crawler';
-import { TRPCError } from '@trpc/server';
-import pMap from 'p-map';
 import { z } from 'zod';
 
-import { toolsEnv } from '@/config/tools';
 import { isServerMode } from '@/const/version';
 import { passwordProcedure } from '@/libs/trpc/edge';
 import { authedProcedure, router } from '@/libs/trpc/lambda';
-import { SearXNGClient } from '@/server/modules/SearXNG';
-import { SEARCH_SEARXNG_NOT_CONFIG } from '@/types/tool/search';
+import { searchService } from '@/server/services/search';
 
 // TODO: password procedure 未来的处理方式可能要思考下
 const searchProcedure = isServerMode ? authedProcedure : passwordProcedure;
@@ -22,24 +17,7 @@ export const searchRouter = router({
       }),
     )
     .mutation(async ({ input }) => {
-      const envString = toolsEnv.CRAWLER_IMPLS || '';
-
-      // 处理全角逗号和多余空格
-      let envValue = envString.replaceAll('，', ',').trim();
-
-      const impls = envValue.split(',').filter(Boolean);
-
-      const crawler = new Crawler({ impls });
-
-      const results = await pMap(
-        input.urls,
-        async (url) => {
-          return await crawler.crawl({ impls: input.impls, url });
-        },
-        { concurrency: 3 },
-      );
-
-      return { results };
+      return searchService.crawlPages(input);
     }),
 
   query: searchProcedure
@@ -56,25 +34,6 @@ export const searchRouter = router({
       }),
     )
     .query(async ({ input }) => {
-      if (!toolsEnv.SEARXNG_URL) {
-        throw new TRPCError({ code: 'NOT_IMPLEMENTED', message: SEARCH_SEARXNG_NOT_CONFIG });
-      }
-
-      const client = new SearXNGClient(toolsEnv.SEARXNG_URL);
-
-      try {
-        return await client.search(input.query, {
-          categories: input.optionalParams?.searchCategories,
-          engines: input.optionalParams?.searchEngines,
-          time_range: input.optionalParams?.searchTimeRange,
-        });
-      } catch (e) {
-        console.error(e);
-
-        throw new TRPCError({
-          code: 'SERVICE_UNAVAILABLE',
-          message: (e as Error).message,
-        });
-      }
+      return await searchService.query(input.query, input.optionalParams);
     }),
 });

package/src/server/services/search/impls/index.ts

@@ -0,0 +1,30 @@
+import { Search1APIImpl } from './search1api';
+import { SearXNGImpl } from './searxng';
+import { SearchServiceImpl } from './type';
+
+/**
+ * Available search service implementations
+ */
+export enum SearchImplType {
+  SearXNG = 'searxng',
+  Search1API = 'search1api',
+}
+
+/**
+ * Create a search service implementation instance
+ */
+export const createSearchServiceImpl = (
+  type: SearchImplType = SearchImplType.SearXNG,
+): SearchServiceImpl => {
+  switch (type) {
+    case SearchImplType.SearXNG: {
+      return new SearXNGImpl();
+    }
+
+    default: {
+      return new Search1APIImpl();
+    }
+  }
+};
+
+export type { SearchServiceImpl } from './type';

package/src/server/services/search/impls/search1api/index.ts

@@ -0,0 +1,154 @@
+import { TRPCError } from '@trpc/server';
+import debug from 'debug';
+import urlJoin from 'url-join';
+
+import { SearchParams, UniformSearchResponse, UniformSearchResult } from '@/types/tool/search';
+
+import { SearchServiceImpl } from '../type';
+import { Search1ApiResponse } from './type';
+
+interface Search1APIQueryParams {
+  crawl_results?: 0 | 1;
+  exclude_sites?: string[];
+  image?: boolean;
+  include_sites?: string[];
+  language?: string;
+  max_results: number;
+  query: string;
+  search_service?: string;
+  time_range?: string;
+}
+
+const log = debug('lobe-search:search1api');
+
+/**
+ * Search1API implementation of the search service
+ * Primarily used for web crawling
+ */
+export class Search1APIImpl implements SearchServiceImpl {
+  private get apiKey(): string | undefined {
+    return process.env.SEARCH1API_SEARCH_API_KEY || process.env.SEARCH1API_API_KEY;
+  }
+
+  private get baseUrl(): string {
+    // Assuming the base URL is consistent with the crawl endpoint
+    return 'https://api.search1api.com';
+  }
+
+  async query(query: string, params: SearchParams = {}): Promise<UniformSearchResponse> {
+    log('Starting Search1API query with query: "%s", params: %o', query, params);
+    const endpoint = urlJoin(this.baseUrl, '/search');
+
+    const { searchEngines } = params;
+
+    const defaultQueryParams: Search1APIQueryParams = {
+      crawl_results: 0, // 默认不做抓取
+      image: false,
+      max_results: 15, // Default max results
+      query,
+    };
+
+    let body: Search1APIQueryParams[] = [
+      {
+        ...defaultQueryParams,
+        time_range:
+          params?.searchTimeRange && params.searchTimeRange !== 'anytime'
+            ? params.searchTimeRange
+            : undefined,
+      },
+    ];
+
+    if (searchEngines && searchEngines.length > 0) {
+      body = searchEngines.map((searchEngine) => ({
+        ...defaultQueryParams,
+
+        max_results: parseInt((20 / searchEngines.length).toFixed(0)),
+        search_service: searchEngine,
+        time_range:
+          params?.searchTimeRange && params.searchTimeRange !== 'anytime'
+            ? params.searchTimeRange
+            : undefined,
+      }));
+    }
+
+    // Note: Other SearchParams like searchCategories, searchEngines (beyond the first one)
+    // and Search1API specific params like include_sites, exclude_sites, language
+    // are not currently mapped.
+
+    log('Constructed request body: %o', body);
+
+    let response: Response;
+    const startAt = Date.now();
+    let costTime = 0;
+    try {
+      log('Sending request to endpoint: %s', endpoint);
+      response = await fetch(endpoint, {
+        body: JSON.stringify(body),
+        headers: {
+          'Authorization': this.apiKey ? `Bearer ${this.apiKey}` : '',
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+      });
+      log('Received response with status: %d', response.status);
+      costTime = Date.now() - startAt;
+    } catch (error) {
+      log.extend('error')('Search1API fetch error: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'SERVICE_UNAVAILABLE',
+        message: 'Failed to connect to Search1API.',
+      });
+    }
+
+    if (!response.ok) {
+      const errorBody = await response.text();
+      log.extend('error')(
+        `Search1API request failed with status ${response.status}: %s`,
+        errorBody.length > 200 ? `${errorBody.slice(0, 200)}...` : errorBody,
+      );
+      throw new TRPCError({
+        cause: errorBody,
+        code: 'SERVICE_UNAVAILABLE',
+        message: `Search1API request failed: ${response.statusText}`,
+      });
+    }
+
+    try {
+      const search1ApiResponse = (await response.json()) as Search1ApiResponse[]; // Use a specific type if defined elsewhere
+
+      log('Parsed Search1API response: %o', search1ApiResponse);
+
+      const mappedResults = search1ApiResponse.flatMap((response) => {
+        // Map Search1API response to SearchResponse
+        return (response.results || []).map(
+          (result): UniformSearchResult => ({
+            category: 'general', // Default category
+            content: result.content || result.snippet || '', // Prioritize content, fallback to snippet
+            engines: [response.searchParameters?.search_service || ''],
+            parsedUrl: result.link ? new URL(result.link).hostname : '', // Basic URL parsing
+            score: 1, // Default score
+            title: result.title || '',
+            url: result.link,
+          }),
+        );
+      });
+
+      log('Mapped %d results to SearchResult format', mappedResults.length);
+
+      return {
+        costTime,
+        query: query,
+        resultNumbers: mappedResults.length,
+        results: mappedResults,
+      };
+    } catch (error) {
+      log.extend('error')('Error parsing Search1API response: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'INTERNAL_SERVER_ERROR',
+        message: 'Failed to parse Search1API response.',
+      });
+    }
+  }
+}