@lobehub/chat 1.80.5 → 1.81.0

package/src/server/services/search/impls/search1api/type.ts

@@ -0,0 +1,81 @@
+/**
+ * The query you want to ask
+ */
+export type Query = string;
+
+export const SEARCH1API_SUPPORT_SEARCH_SERVICE = [
+  'google',
+  'bing',
+  'duckduckgo',
+  'yahoo',
+  'youtube',
+  'x',
+  'reddit',
+  'github',
+  'arxiv',
+  'wechat',
+  'bilibili',
+  'imdb',
+  'wikipedia',
+] as const;
+
+/**
+ * The search service you want to choose
+ */
+export type SearchService = (typeof SEARCH1API_SUPPORT_SEARCH_SERVICE)[number];
+
+/**
+ * The results you want to have
+ */
+export type MaxResults = number;
+/**
+ * The results you want to crawl
+ */
+export type CrawlResults = number;
+/**
+ * Search including image urls
+ */
+export type Image = boolean;
+/**
+ * List of websites to include in search results
+ */
+export type IncludeSites = string[];
+/**
+ * List of websites to exclude from search results
+ */
+export type ExcludeSites = string[];
+/**
+ * The language preference for search results (e.g., 'en', 'zh-CN', 'fr'). Uses standard language codes.
+ */
+export type Language = string;
+/**
+ * Limit search results to a specific time range
+ */
+export type TimeRange = 'day' | 'month' | 'year';
+
+export interface Search1ApiSearchParameters {
+  crawl_results?: CrawlResults;
+  exclude_sites?: ExcludeSites;
+  image?: Image;
+  include_sites?: IncludeSites;
+  language?: Language;
+  max_results?: MaxResults;
+  query: Query;
+  search_service?: SearchService;
+  time_range?: TimeRange;
+}
+
+// Define the Search1API specific response structure based on user input
+// Ideally, this would live in a dedicated types file (e.g., src/types/tool/search/search1api.ts)
+interface Search1ApiResult {
+  content?: string;
+  link: string;
+  snippet?: string;
+  title?: string;
+}
+
+export interface Search1ApiResponse {
+  // Keeping this generic for now
+  results?: Search1ApiResult[];
+  searchParameters?: Search1ApiSearchParameters;
+}

package/src/server/{modules/SearXNG.ts → services/search/impls/searxng/client.ts}

@@ -1,7 +1,34 @@
 import qs from 'query-string';
 import urlJoin from 'url-join';
 
-import { SearchResponse } from '@/types/tool/search';
+export interface SearXNGSearchResult {
+  category: string;
+  content?: string;
+  engine: string;
+  engines: string[];
+  iframe_src?: string;
+  img_src?: string;
+  parsed_url: string[];
+  positions: number[];
+  publishedDate?: string | null;
+  score: number;
+  template: string;
+  thumbnail?: string | null;
+  thumbnail_src?: string | null;
+  title: string;
+  url: string;
+}
+
+export interface SearXNGSearchResponse {
+  answers: any[];
+  corrections: any[];
+  infoboxes: any[];
+  number_of_results: number;
+  query: string;
+  results: SearXNGSearchResult[];
+  suggestions: string[];
+  unresponsive_engines: any[];
+}
 
 export class SearXNGClient {
   private baseUrl: string;
@@ -10,7 +37,10 @@ export class SearXNGClient {
     this.baseUrl = baseUrl;
   }
 
-  async search(query: string, optionalParams: Record<string, any> = {}): Promise<SearchResponse> {
+  async search(
+    query: string,
+    optionalParams: Record<string, any> = {},
+  ): Promise<SearXNGSearchResponse> {
     try {
       const { time_range, ...otherParams } = optionalParams;
 

package/src/server/{routers/tools/__tests__ → services/search/impls/searxng}/fixtures/searXNG.ts

@@ -1,6 +1,6 @@
-import { SearchResponse } from '@/types/tool/search';
+import { SearXNGSearchResponse } from '../client';
 
-export const hetongxue: SearchResponse = {
+export const hetongxue: SearXNGSearchResponse = {
   answers: [
     '老师好我叫何同学. 目标是做有意思的视频 合作请联系：xhaxx1123@163.com. 【何同学】我拍了一张600万人的合影...共计2条视频，包括：正片、教程等，UP主更多精彩视频，请关注UP账号。.',
   ],

package/src/server/services/search/impls/searxng/index.test.ts

@@ -0,0 +1,26 @@
+// @vitest-environment node
+import { describe, expect, it, vi } from 'vitest';
+
+import { SearXNGClient } from './client';
+import { hetongxue } from './fixtures/searXNG';
+import { SearXNGImpl } from './index';
+
+vi.mock('@/config/tools', () => ({
+  toolsEnv: {
+    SEARXNG_URL: 'https://demo.com',
+  },
+}));
+
+describe('SearXNGImpl', () => {
+  describe('query', () => {
+    it('搜索结果超过10个', async () => {
+      vi.spyOn(SearXNGClient.prototype, 'search').mockResolvedValueOnce(hetongxue);
+
+      const searchImpl = new SearXNGImpl();
+      const results = await searchImpl.query('何同学');
+
+      // Assert
+      expect(results.results.length).toEqual(43);
+    });
+  });
+});

package/src/server/services/search/impls/searxng/index.ts

@@ -0,0 +1,62 @@
+import { TRPCError } from '@trpc/server';
+
+import { toolsEnv } from '@/config/tools';
+import { SearXNGClient } from '@/server/services/search/impls/searxng/client';
+import { SEARCH_SEARXNG_NOT_CONFIG, UniformSearchResponse } from '@/types/tool/search';
+
+import { SearchServiceImpl } from '../type';
+
+/**
+ * SearXNG implementation of the search service
+ */
+export class SearXNGImpl implements SearchServiceImpl {
+  async query(
+    query: string,
+    params?: {
+      searchCategories?: string[];
+      searchEngines?: string[];
+      searchTimeRange?: string;
+    },
+  ): Promise<UniformSearchResponse> {
+    if (!toolsEnv.SEARXNG_URL) {
+      throw new TRPCError({ code: 'NOT_IMPLEMENTED', message: SEARCH_SEARXNG_NOT_CONFIG });
+    }
+
+    const client = new SearXNGClient(toolsEnv.SEARXNG_URL);
+
+    try {
+      let costTime = 0;
+      const startAt = Date.now();
+      const data = await client.search(query, {
+        categories: params?.searchCategories,
+        engines: params?.searchEngines,
+        time_range: params?.searchTimeRange,
+      });
+      costTime = Date.now() - startAt;
+
+      return {
+        costTime,
+        query,
+        resultNumbers: data.number_of_results,
+        results: data.results.map((item) => ({
+          category: item.category,
+          content: item.content!,
+          engines: item.engines,
+          parsedUrl: item.url ? new URL(item.url).hostname : '',
+          publishedDate: item.publishedDate || undefined,
+          score: item.score,
+          thumbnail: item.thumbnail || undefined,
+          title: item.title,
+          url: item.url,
+        })),
+      };
+    } catch (e) {
+      console.error(e);
+
+      throw new TRPCError({
+        code: 'SERVICE_UNAVAILABLE',
+        message: (e as Error).message,
+      });
+    }
+  }
+}

package/src/server/services/search/impls/type.ts

@@ -0,0 +1,11 @@
+import { SearchParams, UniformSearchResponse } from '@/types/tool/search';
+
+/**
+ * Search service implementation interface
+ */
+export interface SearchServiceImpl {
+  /**
+   * Query for search results
+   */
+  query(query: string, params?: SearchParams): Promise<UniformSearchResponse>;
+}

package/src/server/services/search/index.ts

@@ -0,0 +1,59 @@
+import { CrawlImplType, Crawler } from '@lobechat/web-crawler';
+import pMap from 'p-map';
+
+import { toolsEnv } from '@/config/tools';
+import { SearchParams } from '@/types/tool/search';
+
+import { SearchImplType, SearchServiceImpl, createSearchServiceImpl } from './impls';
+
+const parseImplEnv = (envString: string = '') => {
+  // 处理全角逗号和多余空格
+  const envValue = envString.replaceAll('，', ',').trim();
+  return envValue.split(',').filter(Boolean);
+};
+
+/**
+ * Search service class
+ * Uses different implementations for different search operations
+ */
+export class SearchService {
+  private searchImpl: SearchServiceImpl;
+
+  private get crawlerImpls() {
+    return parseImplEnv(toolsEnv.CRAWLER_IMPLS);
+  }
+
+  constructor() {
+    const impls = this.searchImpls;
+    // TODO: need use turn mode
+    this.searchImpl = createSearchServiceImpl(impls.length > 0 ? impls[0] : undefined);
+  }
+
+  async crawlPages(input: { impls?: CrawlImplType[]; urls: string[] }) {
+    const crawler = new Crawler({ impls: this.crawlerImpls });
+
+    const results = await pMap(
+      input.urls,
+      async (url) => {
+        return await crawler.crawl({ impls: input.impls, url });
+      },
+      { concurrency: 3 },
+    );
+
+    return { results };
+  }
+
+  private get searchImpls() {
+    return parseImplEnv(toolsEnv.SEARCH_PROVIDERS) as SearchImplType[];
+  }
+
+  /**
+   * Query for search results
+   */
+  async query(query: string, params?: SearchParams) {
+    return this.searchImpl.query(query, params);
+  }
+}
+
+// Add a default exported instance for convenience
+export const searchService = new SearchService();

package/src/store/chat/slices/builtinTool/actions/index.ts

@@ -3,7 +3,7 @@ import { StateCreator } from 'zustand/vanilla';
 import { ChatStore } from '@/store/chat/store';
 
 import { ChatDallEAction, dalleSlice } from './dalle';
-import { SearchAction, searchSlice } from './searXNG';
+import { SearchAction, searchSlice } from './search';
 
 export interface ChatBuiltinToolAction extends ChatDallEAction, SearchAction {}
 

package/src/store/chat/slices/builtinTool/actions/{searXNG.test.ts → search.test.ts}

@@ -6,7 +6,7 @@ import { useChatStore } from '@/store/chat';
 import { chatSelectors } from '@/store/chat/selectors';
 import { CRAWL_CONTENT_LIMITED_COUNT } from '@/tools/web-browsing/const';
 import { ChatMessage } from '@/types/message';
-import { SearchContent, SearchQuery, SearchResponse } from '@/types/tool/search';
+import { SearchContent, SearchQuery, UniformSearchResponse } from '@/types/tool/search';
 
 // Mock services
 vi.mock('@/services/search', () => ({
@@ -22,7 +22,7 @@ vi.mock('@/store/chat/selectors', () => ({
   },
 }));
 
-describe('searXNG actions', () => {
+describe('search actions', () => {
   beforeEach(() => {
     vi.clearAllMocks();
     useChatStore.setState({
@@ -39,47 +39,38 @@ describe('searXNG actions', () => {
     });
   });
 
-  describe('searchWithSearXNG', () => {
+  describe('search', () => {
     it('should handle successful search', async () => {
-      const mockResponse: SearchResponse = {
+      const mockResponse: UniformSearchResponse = {
         results: [
           {
             title: 'Test Result',
             content: 'Test Content',
             url: 'https://test.com',
             category: 'general',
-            engine: 'google',
             engines: ['google'],
-            parsed_url: ['test.com'],
-            positions: [1],
+            parsedUrl: 'test.com',
             score: 1,
-            template: 'default',
           },
         ],
-        answers: [],
-        corrections: [],
-        infoboxes: [],
-        number_of_results: 1,
+        costTime: 1,
+        resultNumbers: 1,
         query: 'test',
-        suggestions: [],
-        unresponsive_engines: [],
       };
 
       (searchService.search as Mock).mockResolvedValue(mockResponse);
 
       const { result } = renderHook(() => useChatStore());
-      const { searchWithSearXNG } = result.current;
+      const { search } = result.current;
 
       const messageId = 'test-message-id';
       const query: SearchQuery = {
-        optionalParams: {
-          searchEngines: ['google'],
-        },
+        searchEngines: ['google'],
         query: 'test query',
       };
 
       await act(async () => {
-        await searchWithSearXNG(messageId, query);
+        await search(messageId, query);
       });
 
       const expectedContent: SearchContent[] = [
@@ -101,39 +92,28 @@ describe('searXNG actions', () => {
     });
 
     it('should handle empty search results and retry with default engine', async () => {
-      const emptyResponse: SearchResponse = {
+      const emptyResponse: UniformSearchResponse = {
         results: [],
-        answers: [],
-        corrections: [],
-        infoboxes: [],
-        number_of_results: 0,
+        costTime: 1,
+        resultNumbers: 0,
         query: 'test',
-        suggestions: [],
-        unresponsive_engines: [],
       };
 
-      const retryResponse: SearchResponse = {
+      const retryResponse: UniformSearchResponse = {
         results: [
           {
             title: 'Retry Result',
             content: 'Retry Content',
             url: 'https://retry.com',
             category: 'general',
-            engine: 'google',
             engines: ['google'],
-            parsed_url: ['retry.com'],
-            positions: [1],
+            parsedUrl: 'retry.com',
             score: 1,
-            template: 'default',
           },
         ],
-        answers: [],
-        corrections: [],
-        infoboxes: [],
-        number_of_results: 1,
+        costTime: 1,
+        resultNumbers: 1,
         query: 'test',
-        suggestions: [],
-        unresponsive_engines: [],
       };
 
       (searchService.search as Mock)
@@ -142,19 +122,17 @@ describe('searXNG actions', () => {
         .mockResolvedValueOnce(retryResponse);
 
       const { result } = renderHook(() => useChatStore());
-      const { searchWithSearXNG } = result.current;
+      const { search } = result.current;
 
       const messageId = 'test-message-id';
       const query: SearchQuery = {
-        optionalParams: {
-          searchEngines: ['custom-engine'],
-          searchTimeRange: 'year',
-        },
+        searchEngines: ['custom-engine'],
+        searchTimeRange: 'year',
         query: 'test query',
       };
 
       await act(async () => {
-        await searchWithSearXNG(messageId, query);
+        await search(messageId, query);
       });
 
       expect(searchService.search).toHaveBeenCalledTimes(3);
@@ -166,9 +144,6 @@ describe('searXNG actions', () => {
         searchTimeRange: 'year',
       });
       expect(result.current.updatePluginArguments).toHaveBeenCalledWith(messageId, {
-        optionalParams: {
-          searchTimeRange: 'year',
-        },
         query: 'test query',
       });
       expect(searchService.search).toHaveBeenNthCalledWith(3, 'test query');
@@ -183,7 +158,7 @@ describe('searXNG actions', () => {
       (searchService.search as Mock).mockRejectedValue(error);
 
       const { result } = renderHook(() => useChatStore());
-      const { searchWithSearXNG } = result.current;
+      const { search } = result.current;
 
       const messageId = 'test-message-id';
       const query: SearchQuery = {
@@ -191,7 +166,7 @@ describe('searXNG actions', () => {
       };
 
       await act(async () => {
-        await searchWithSearXNG(messageId, query);
+        await search(messageId, query);
       });
 
       expect(result.current.internal_updateMessagePluginError).toHaveBeenCalledWith(messageId, {
@@ -271,8 +246,8 @@ describe('searXNG actions', () => {
   describe('reSearchWithSearXNG', () => {
     it('should update arguments and perform search', async () => {
       const { result } = renderHook(() => useChatStore());
-      const spy = vi.spyOn(result.current, 'searchWithSearXNG');
-      const { reSearchWithSearXNG } = result.current;
+      const spy = vi.spyOn(result.current, 'search');
+      const { triggerSearchAgain } = result.current;
 
       const messageId = 'test-message-id';
       const query: SearchQuery = {
@@ -280,7 +255,7 @@ describe('searXNG actions', () => {
       };
 
       await act(async () => {
-        await reSearchWithSearXNG(messageId, query, { aiSummary: true });
+        await triggerSearchAgain(messageId, query, { aiSummary: true });
       });
 
       expect(result.current.updatePluginArguments).toHaveBeenCalledWith(messageId, query);
@@ -314,10 +289,10 @@ describe('searXNG actions', () => {
       );
 
       const { result } = renderHook(() => useChatStore());
-      const { saveSearXNGSearchResult } = result.current;
+      const { saveSearchResult } = result.current;
 
       await act(async () => {
-        await saveSearXNGSearchResult(messageId);
+        await saveSearchResult(messageId);
       });
 
       expect(result.current.internal_createMessage).toHaveBeenCalledWith(
@@ -343,10 +318,10 @@ describe('searXNG actions', () => {
       vi.spyOn(chatSelectors, 'getMessageById').mockImplementation(() => () => undefined);
 
       const { result } = renderHook(() => useChatStore());
-      const { saveSearXNGSearchResult } = result.current;
+      const { saveSearchResult } = result.current;
 
       await act(async () => {
-        await saveSearXNGSearchResult('non-existent-id');
+        await saveSearchResult('non-existent-id');
       });
 
       expect(result.current.internal_createMessage).not.toHaveBeenCalled();

package/src/store/chat/slices/builtinTool/actions/{searXNG.ts → search.ts}

@@ -9,7 +9,7 @@ import {
   SEARCH_SEARXNG_NOT_CONFIG,
   SearchContent,
   SearchQuery,
-  SearchResponse,
+  UniformSearchResponse,
 } from '@/types/tool/search';
 import { nanoid } from '@/utils/uuid';
 
@@ -24,23 +24,19 @@ export interface SearchAction {
     params: { url: string },
     aiSummary?: boolean,
   ) => Promise<boolean | undefined>;
+  saveSearchResult: (id: string) => Promise<void>;
+  search: (id: string, data: SearchQuery, aiSummary?: boolean) => Promise<void | boolean>;
+  togglePageContent: (url: string) => void;
+  toggleSearchLoading: (id: string, loading: boolean) => void;
   /**
    * 重新发起搜索
    * @description 会更新插件的 arguments 参数，然后再次搜索
    */
-  reSearchWithSearXNG: (
+  triggerSearchAgain: (
     id: string,
     data: SearchQuery,
     options?: { aiSummary: boolean },
   ) => Promise<void>;
-  saveSearXNGSearchResult: (id: string) => Promise<void>;
-  searchWithSearXNG: (
-    id: string,
-    data: SearchQuery,
-    aiSummary?: boolean,
-  ) => Promise<void | boolean>;
-  togglePageContent: (url: string) => void;
-  toggleSearchLoading: (id: string, loading: boolean) => void;
 }
 
 export const searchSlice: StateCreator<
@@ -91,14 +87,7 @@ export const searchSlice: StateCreator<
     return await crawlMultiPages(id, { urls: [params.url] }, aiSummary);
   },
 
-  reSearchWithSearXNG: async (id, data, options) => {
-    get().toggleSearchLoading(id, true);
-    await get().updatePluginArguments(id, data);
-
-    await get().searchWithSearXNG(id, data, options?.aiSummary);
-  },
-
-  saveSearXNGSearchResult: async (id) => {
+  saveSearchResult: async (id) => {
     const message = chatSelectors.getMessageById(id)(get());
     if (!message || !message.plugin) return;
 
@@ -138,34 +127,32 @@ export const searchSlice: StateCreator<
     // 将新创建的 tool message 激活
     openToolUI(newMessageId, message.plugin.identifier);
   },
-  searchWithSearXNG: async (id, params, aiSummary = true) => {
+
+  search: async (id, { query, ...params }, aiSummary = true) => {
     get().toggleSearchLoading(id, true);
-    let data: SearchResponse | undefined;
+    let data: UniformSearchResponse | undefined;
     try {
       // 首次查询
-      data = await searchService.search(params.query, params.optionalParams);
+      data = await searchService.search(query, params);
 
       // 如果没有搜索到结果，则执行第一次重试（移除搜索引擎限制）
       if (
         data?.results.length === 0 &&
-        params.optionalParams?.searchEngines &&
-        params.optionalParams?.searchEngines?.length > 0
+        params?.searchEngines &&
+        params?.searchEngines?.length > 0
       ) {
         const paramsExcludeSearchEngines = {
           ...params,
-          optionalParams: {
-            ...params.optionalParams,
-            searchEngines: undefined,
-          },
+          searchEngines: undefined,
         };
-        data = await searchService.search(params.query, paramsExcludeSearchEngines.optionalParams);
+        data = await searchService.search(query, paramsExcludeSearchEngines);
         get().updatePluginArguments(id, paramsExcludeSearchEngines);
       }
 
       // 如果仍然没有搜索到结果，则执行第二次重试（移除所有限制）
       if (data?.results.length === 0) {
-        data = await searchService.search(params.query);
-        get().updatePluginArguments(id, { ...params, optionalParams: undefined });
+        data = await searchService.search(query);
+        get().updatePluginArguments(id, { query });
       }
 
       await get().updatePluginState(id, data);
@@ -197,7 +184,7 @@ export const searchSlice: StateCreator<
       url: item.url,
       ...(item.content && { content: item.content }),
       ...(item.publishedDate && { publishedDate: item.publishedDate }),
-      ...(item.img_src && { img_src: item.img_src }),
+      ...(item.imgSrc && { imgSrc: item.imgSrc }),
       ...(item.thumbnail && { thumbnail: item.thumbnail }),
     }));
 
@@ -209,7 +196,6 @@ export const searchSlice: StateCreator<
     // 如果 aiSummary 为 true，则会自动触发总结
     return aiSummary;
   },
-
   togglePageContent: (url) => {
     set({ activePageContentUrl: url });
   },
@@ -221,4 +207,11 @@ export const searchSlice: StateCreator<
       `toggleSearchLoading/${loading ? 'start' : 'end'}`,
     );
   },
+
+  triggerSearchAgain: async (id, data, options) => {
+    get().toggleSearchLoading(id, true);
+    await get().updatePluginArguments(id, data);
+
+    await get().search(id, data, options?.aiSummary);
+  },
 });

package/src/tools/web-browsing/Portal/Search/Footer.tsx

@@ -12,7 +12,7 @@ const Footer = () => {
     chatPortalSelectors.toolMessageId(s),
     chatSelectors.isAIGenerating(s),
     s.triggerAIMessage,
-    s.saveSearXNGSearchResult,
+    s.saveSearchResult,
   ]);
 
   const { t } = useTranslation('tool');

package/src/tools/web-browsing/Portal/Search/ResultList/SearchItem/TitleExtra.tsx

@@ -9,7 +9,7 @@ import { EngineAvatarGroup } from '@/tools/web-browsing/components/EngineAvatar'
 import CategoryAvatar from './CategoryAvatar';
 
 interface TitleExtraProps {
-  category: string;
+  category?: string;
   engines: string[];
   highlight?: boolean;
   score: number;
@@ -35,7 +35,7 @@ const TitleExtra = memo<TitleExtraProps>(({ category, score, highlight, engines
           </Typography.Text>
         )}
       </Tooltip>
-      <CategoryAvatar category={category} />
+      <CategoryAvatar category={category || 'general'} />
     </Flexbox>
   );
 });