@lobehub/chat 1.80.4 → 1.81.0

package/src/server/services/search/impls/index.ts

@@ -0,0 +1,30 @@
+import { Search1APIImpl } from './search1api';
+import { SearXNGImpl } from './searxng';
+import { SearchServiceImpl } from './type';
+
+/**
+ * Available search service implementations
+ */
+export enum SearchImplType {
+  SearXNG = 'searxng',
+  Search1API = 'search1api',
+}
+
+/**
+ * Create a search service implementation instance
+ */
+export const createSearchServiceImpl = (
+  type: SearchImplType = SearchImplType.SearXNG,
+): SearchServiceImpl => {
+  switch (type) {
+    case SearchImplType.SearXNG: {
+      return new SearXNGImpl();
+    }
+
+    default: {
+      return new Search1APIImpl();
+    }
+  }
+};
+
+export type { SearchServiceImpl } from './type';

package/src/server/services/search/impls/search1api/index.ts

@@ -0,0 +1,154 @@
+import { TRPCError } from '@trpc/server';
+import debug from 'debug';
+import urlJoin from 'url-join';
+
+import { SearchParams, UniformSearchResponse, UniformSearchResult } from '@/types/tool/search';
+
+import { SearchServiceImpl } from '../type';
+import { Search1ApiResponse } from './type';
+
+interface Search1APIQueryParams {
+  crawl_results?: 0 | 1;
+  exclude_sites?: string[];
+  image?: boolean;
+  include_sites?: string[];
+  language?: string;
+  max_results: number;
+  query: string;
+  search_service?: string;
+  time_range?: string;
+}
+
+const log = debug('lobe-search:search1api');
+
+/**
+ * Search1API implementation of the search service
+ * Primarily used for web crawling
+ */
+export class Search1APIImpl implements SearchServiceImpl {
+  private get apiKey(): string | undefined {
+    return process.env.SEARCH1API_SEARCH_API_KEY || process.env.SEARCH1API_API_KEY;
+  }
+
+  private get baseUrl(): string {
+    // Assuming the base URL is consistent with the crawl endpoint
+    return 'https://api.search1api.com';
+  }
+
+  async query(query: string, params: SearchParams = {}): Promise<UniformSearchResponse> {
+    log('Starting Search1API query with query: "%s", params: %o', query, params);
+    const endpoint = urlJoin(this.baseUrl, '/search');
+
+    const { searchEngines } = params;
+
+    const defaultQueryParams: Search1APIQueryParams = {
+      crawl_results: 0, // 默认不做抓取
+      image: false,
+      max_results: 15, // Default max results
+      query,
+    };
+
+    let body: Search1APIQueryParams[] = [
+      {
+        ...defaultQueryParams,
+        time_range:
+          params?.searchTimeRange && params.searchTimeRange !== 'anytime'
+            ? params.searchTimeRange
+            : undefined,
+      },
+    ];
+
+    if (searchEngines && searchEngines.length > 0) {
+      body = searchEngines.map((searchEngine) => ({
+        ...defaultQueryParams,
+
+        max_results: parseInt((20 / searchEngines.length).toFixed(0)),
+        search_service: searchEngine,
+        time_range:
+          params?.searchTimeRange && params.searchTimeRange !== 'anytime'
+            ? params.searchTimeRange
+            : undefined,
+      }));
+    }
+
+    // Note: Other SearchParams like searchCategories, searchEngines (beyond the first one)
+    // and Search1API specific params like include_sites, exclude_sites, language
+    // are not currently mapped.
+
+    log('Constructed request body: %o', body);
+
+    let response: Response;
+    const startAt = Date.now();
+    let costTime = 0;
+    try {
+      log('Sending request to endpoint: %s', endpoint);
+      response = await fetch(endpoint, {
+        body: JSON.stringify(body),
+        headers: {
+          'Authorization': this.apiKey ? `Bearer ${this.apiKey}` : '',
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+      });
+      log('Received response with status: %d', response.status);
+      costTime = Date.now() - startAt;
+    } catch (error) {
+      log.extend('error')('Search1API fetch error: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'SERVICE_UNAVAILABLE',
+        message: 'Failed to connect to Search1API.',
+      });
+    }
+
+    if (!response.ok) {
+      const errorBody = await response.text();
+      log.extend('error')(
+        `Search1API request failed with status ${response.status}: %s`,
+        errorBody.length > 200 ? `${errorBody.slice(0, 200)}...` : errorBody,
+      );
+      throw new TRPCError({
+        cause: errorBody,
+        code: 'SERVICE_UNAVAILABLE',
+        message: `Search1API request failed: ${response.statusText}`,
+      });
+    }
+
+    try {
+      const search1ApiResponse = (await response.json()) as Search1ApiResponse[]; // Use a specific type if defined elsewhere
+
+      log('Parsed Search1API response: %o', search1ApiResponse);
+
+      const mappedResults = search1ApiResponse.flatMap((response) => {
+        // Map Search1API response to SearchResponse
+        return (response.results || []).map(
+          (result): UniformSearchResult => ({
+            category: 'general', // Default category
+            content: result.content || result.snippet || '', // Prioritize content, fallback to snippet
+            engines: [response.searchParameters?.search_service || ''],
+            parsedUrl: result.link ? new URL(result.link).hostname : '', // Basic URL parsing
+            score: 1, // Default score
+            title: result.title || '',
+            url: result.link,
+          }),
+        );
+      });
+
+      log('Mapped %d results to SearchResult format', mappedResults.length);
+
+      return {
+        costTime,
+        query: query,
+        resultNumbers: mappedResults.length,
+        results: mappedResults,
+      };
+    } catch (error) {
+      log.extend('error')('Error parsing Search1API response: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'INTERNAL_SERVER_ERROR',
+        message: 'Failed to parse Search1API response.',
+      });
+    }
+  }
+}

package/src/server/services/search/impls/search1api/type.ts

@@ -0,0 +1,81 @@
+/**
+ * The query you want to ask
+ */
+export type Query = string;
+
+export const SEARCH1API_SUPPORT_SEARCH_SERVICE = [
+  'google',
+  'bing',
+  'duckduckgo',
+  'yahoo',
+  'youtube',
+  'x',
+  'reddit',
+  'github',
+  'arxiv',
+  'wechat',
+  'bilibili',
+  'imdb',
+  'wikipedia',
+] as const;
+
+/**
+ * The search service you want to choose
+ */
+export type SearchService = (typeof SEARCH1API_SUPPORT_SEARCH_SERVICE)[number];
+
+/**
+ * The results you want to have
+ */
+export type MaxResults = number;
+/**
+ * The results you want to crawl
+ */
+export type CrawlResults = number;
+/**
+ * Search including image urls
+ */
+export type Image = boolean;
+/**
+ * List of websites to include in search results
+ */
+export type IncludeSites = string[];
+/**
+ * List of websites to exclude from search results
+ */
+export type ExcludeSites = string[];
+/**
+ * The language preference for search results (e.g., 'en', 'zh-CN', 'fr'). Uses standard language codes.
+ */
+export type Language = string;
+/**
+ * Limit search results to a specific time range
+ */
+export type TimeRange = 'day' | 'month' | 'year';
+
+export interface Search1ApiSearchParameters {
+  crawl_results?: CrawlResults;
+  exclude_sites?: ExcludeSites;
+  image?: Image;
+  include_sites?: IncludeSites;
+  language?: Language;
+  max_results?: MaxResults;
+  query: Query;
+  search_service?: SearchService;
+  time_range?: TimeRange;
+}
+
+// Define the Search1API specific response structure based on user input
+// Ideally, this would live in a dedicated types file (e.g., src/types/tool/search/search1api.ts)
+interface Search1ApiResult {
+  content?: string;
+  link: string;
+  snippet?: string;
+  title?: string;
+}
+
+export interface Search1ApiResponse {
+  // Keeping this generic for now
+  results?: Search1ApiResult[];
+  searchParameters?: Search1ApiSearchParameters;
+}

package/src/server/{modules/SearXNG.ts → services/search/impls/searxng/client.ts}

@@ -1,7 +1,34 @@
 import qs from 'query-string';
 import urlJoin from 'url-join';
 
-import { SearchResponse } from '@/types/tool/search';
+export interface SearXNGSearchResult {
+  category: string;
+  content?: string;
+  engine: string;
+  engines: string[];
+  iframe_src?: string;
+  img_src?: string;
+  parsed_url: string[];
+  positions: number[];
+  publishedDate?: string | null;
+  score: number;
+  template: string;
+  thumbnail?: string | null;
+  thumbnail_src?: string | null;
+  title: string;
+  url: string;
+}
+
+export interface SearXNGSearchResponse {
+  answers: any[];
+  corrections: any[];
+  infoboxes: any[];
+  number_of_results: number;
+  query: string;
+  results: SearXNGSearchResult[];
+  suggestions: string[];
+  unresponsive_engines: any[];
+}
 
 export class SearXNGClient {
   private baseUrl: string;
@@ -10,7 +37,10 @@ export class SearXNGClient {
     this.baseUrl = baseUrl;
   }
 
-  async search(query: string, optionalParams: Record<string, any> = {}): Promise<SearchResponse> {
+  async search(
+    query: string,
+    optionalParams: Record<string, any> = {},
+  ): Promise<SearXNGSearchResponse> {
     try {
       const { time_range, ...otherParams } = optionalParams;
 

package/src/server/{routers/tools/__tests__ → services/search/impls/searxng}/fixtures/searXNG.ts

@@ -1,6 +1,6 @@
-import { SearchResponse } from '@/types/tool/search';
+import { SearXNGSearchResponse } from '../client';
 
-export const hetongxue: SearchResponse = {
+export const hetongxue: SearXNGSearchResponse = {
   answers: [
     '老师好我叫何同学. 目标是做有意思的视频 合作请联系：xhaxx1123@163.com. 【何同学】我拍了一张600万人的合影...共计2条视频，包括：正片、教程等，UP主更多精彩视频，请关注UP账号。.',
   ],

package/src/server/services/search/impls/searxng/index.test.ts

@@ -0,0 +1,26 @@
+// @vitest-environment node
+import { describe, expect, it, vi } from 'vitest';
+
+import { SearXNGClient } from './client';
+import { hetongxue } from './fixtures/searXNG';
+import { SearXNGImpl } from './index';
+
+vi.mock('@/config/tools', () => ({
+  toolsEnv: {
+    SEARXNG_URL: 'https://demo.com',
+  },
+}));
+
+describe('SearXNGImpl', () => {
+  describe('query', () => {
+    it('搜索结果超过10个', async () => {
+      vi.spyOn(SearXNGClient.prototype, 'search').mockResolvedValueOnce(hetongxue);
+
+      const searchImpl = new SearXNGImpl();
+      const results = await searchImpl.query('何同学');
+
+      // Assert
+      expect(results.results.length).toEqual(43);
+    });
+  });
+});

package/src/server/services/search/impls/searxng/index.ts

@@ -0,0 +1,62 @@
+import { TRPCError } from '@trpc/server';
+
+import { toolsEnv } from '@/config/tools';
+import { SearXNGClient } from '@/server/services/search/impls/searxng/client';
+import { SEARCH_SEARXNG_NOT_CONFIG, UniformSearchResponse } from '@/types/tool/search';
+
+import { SearchServiceImpl } from '../type';
+
+/**
+ * SearXNG implementation of the search service
+ */
+export class SearXNGImpl implements SearchServiceImpl {
+  async query(
+    query: string,
+    params?: {
+      searchCategories?: string[];
+      searchEngines?: string[];
+      searchTimeRange?: string;
+    },
+  ): Promise<UniformSearchResponse> {
+    if (!toolsEnv.SEARXNG_URL) {
+      throw new TRPCError({ code: 'NOT_IMPLEMENTED', message: SEARCH_SEARXNG_NOT_CONFIG });
+    }
+
+    const client = new SearXNGClient(toolsEnv.SEARXNG_URL);
+
+    try {
+      let costTime = 0;
+      const startAt = Date.now();
+      const data = await client.search(query, {
+        categories: params?.searchCategories,
+        engines: params?.searchEngines,
+        time_range: params?.searchTimeRange,
+      });
+      costTime = Date.now() - startAt;
+
+      return {
+        costTime,
+        query,
+        resultNumbers: data.number_of_results,
+        results: data.results.map((item) => ({
+          category: item.category,
+          content: item.content!,
+          engines: item.engines,
+          parsedUrl: item.url ? new URL(item.url).hostname : '',
+          publishedDate: item.publishedDate || undefined,
+          score: item.score,
+          thumbnail: item.thumbnail || undefined,
+          title: item.title,
+          url: item.url,
+        })),
+      };
+    } catch (e) {
+      console.error(e);
+
+      throw new TRPCError({
+        code: 'SERVICE_UNAVAILABLE',
+        message: (e as Error).message,
+      });
+    }
+  }
+}

package/src/server/services/search/impls/type.ts

@@ -0,0 +1,11 @@
+import { SearchParams, UniformSearchResponse } from '@/types/tool/search';
+
+/**
+ * Search service implementation interface
+ */
+export interface SearchServiceImpl {
+  /**
+   * Query for search results
+   */
+  query(query: string, params?: SearchParams): Promise<UniformSearchResponse>;
+}

package/src/server/services/search/index.ts

@@ -0,0 +1,59 @@
+import { CrawlImplType, Crawler } from '@lobechat/web-crawler';
+import pMap from 'p-map';
+
+import { toolsEnv } from '@/config/tools';
+import { SearchParams } from '@/types/tool/search';
+
+import { SearchImplType, SearchServiceImpl, createSearchServiceImpl } from './impls';
+
+const parseImplEnv = (envString: string = '') => {
+  // 处理全角逗号和多余空格
+  const envValue = envString.replaceAll('，', ',').trim();
+  return envValue.split(',').filter(Boolean);
+};
+
+/**
+ * Search service class
+ * Uses different implementations for different search operations
+ */
+export class SearchService {
+  private searchImpl: SearchServiceImpl;
+
+  private get crawlerImpls() {
+    return parseImplEnv(toolsEnv.CRAWLER_IMPLS);
+  }
+
+  constructor() {
+    const impls = this.searchImpls;
+    // TODO: need use turn mode
+    this.searchImpl = createSearchServiceImpl(impls.length > 0 ? impls[0] : undefined);
+  }
+
+  async crawlPages(input: { impls?: CrawlImplType[]; urls: string[] }) {
+    const crawler = new Crawler({ impls: this.crawlerImpls });
+
+    const results = await pMap(
+      input.urls,
+      async (url) => {
+        return await crawler.crawl({ impls: input.impls, url });
+      },
+      { concurrency: 3 },
+    );
+
+    return { results };
+  }
+
+  private get searchImpls() {
+    return parseImplEnv(toolsEnv.SEARCH_PROVIDERS) as SearchImplType[];
+  }
+
+  /**
+   * Query for search results
+   */
+  async query(query: string, params?: SearchParams) {
+    return this.searchImpl.query(query, params);
+  }
+}
+
+// Add a default exported instance for convenience
+export const searchService = new SearchService();

package/src/store/chat/slices/builtinTool/actions/index.ts

@@ -3,7 +3,7 @@ import { StateCreator } from 'zustand/vanilla';
 import { ChatStore } from '@/store/chat/store';
 
 import { ChatDallEAction, dalleSlice } from './dalle';
-import { SearchAction, searchSlice } from './searXNG';
+import { SearchAction, searchSlice } from './search';
 
 export interface ChatBuiltinToolAction extends ChatDallEAction, SearchAction {}