@lobehub/chat 1.90.4 → 1.91.1

package/src/server/services/search/impls/exa/index.ts

@@ -0,0 +1,129 @@
+import { TRPCError } from '@trpc/server';
+import debug from 'debug';
+import urlJoin from 'url-join';
+
+import { SearchParams, UniformSearchResponse, UniformSearchResult } from '@/types/tool/search';
+
+import { SearchServiceImpl } from '../type';
+import { ExaSearchParameters, ExaResponse } from './type';
+
+const log = debug('lobe-search:Exa');
+
+/**
+ * Exa implementation of the search service
+ * Primarily used for web crawling
+ */
+export class ExaImpl implements SearchServiceImpl {
+  private get apiKey(): string | undefined {
+    return process.env.EXA_API_KEY;
+  }
+
+  private get baseUrl(): string {
+    // Assuming the base URL is consistent with the crawl endpoint
+    return 'https://api.exa.ai';
+  }
+
+  async query(query: string, params: SearchParams = {}): Promise<UniformSearchResponse> {
+    log('Starting Exa query with query: "%s", params: %o', query, params);
+    const endpoint = urlJoin(this.baseUrl, '/search');
+
+    const defaultQueryParams: ExaSearchParameters = {
+      numResults: 15,
+      query,
+      type: 'auto',
+    };
+
+    let body: ExaSearchParameters = {
+      ...defaultQueryParams,
+      ...(params?.searchTimeRange && params.searchTimeRange !== 'anytime'
+        ? (() => {
+            const now = Date.now();
+            const days = { day: 1, month: 30, week: 7, year: 365 }[params.searchTimeRange!];
+
+            if (days === undefined) return {};
+
+            return {
+              endPublishedDate: new Date(now).toISOString(),
+              startPublishedDate: new Date(now - days * 86_400 * 1000).toISOString(),
+            };
+          })()
+        : {}),
+      category:
+        // Exa 只支持 news 类型
+        params?.searchCategories?.filter(cat => ['news'].includes(cat))?.[0],
+    };
+
+    log('Constructed request body: %o', body);
+
+    let response: Response;
+    const startAt = Date.now();
+    let costTime = 0;
+    try {
+      log('Sending request to endpoint: %s', endpoint);
+      response = await fetch(endpoint, {
+        body: JSON.stringify(body),
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': this.apiKey ? this.apiKey : '',
+        },
+        method: 'POST',
+      });
+      log('Received response with status: %d', response.status);
+      costTime = Date.now() - startAt;
+    } catch (error) {
+      log.extend('error')('Exa fetch error: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'SERVICE_UNAVAILABLE',
+        message: 'Failed to connect to Exa.',
+      });
+    }
+
+    if (!response.ok) {
+      const errorBody = await response.text();
+      log.extend('error')(
+        `Exa request failed with status ${response.status}: %s`,
+        errorBody.length > 200 ? `${errorBody.slice(0, 200)}...` : errorBody,
+      );
+      throw new TRPCError({
+        cause: errorBody,
+        code: 'SERVICE_UNAVAILABLE',
+        message: `Exa request failed: ${response.statusText}`,
+      });
+    }
+
+    try {
+      const exaResponse = (await response.json()) as ExaResponse;
+
+      log('Parsed Exa response: %o', exaResponse);
+
+      const mappedResults = (exaResponse.results || []).map(
+        (result): UniformSearchResult => ({
+          category: body.category || 'general', // Default category
+          content: result.text || '', // Prioritize content, fallback to snippet
+          engines: ['exa'], // Use 'exa' as the engine name
+          parsedUrl: result.url ? new URL(result.url).hostname : '', // Basic URL parsing
+          score: result.score || 0, // Default score to 0 if undefined
+          title: result.title || '',
+          url: result.url,
+        }),
+      );
+
+      log('Mapped %d results to SearchResult format', mappedResults.length);
+
+      return {
+        costTime,
+        query: query,
+        resultNumbers: mappedResults.length,
+        results: mappedResults,
+      };
+    } catch (error) {
+      log.extend('error')('Error parsing Exa response: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'INTERNAL_SERVER_ERROR',
+        message: 'Failed to parse Exa response.',
+      });
+    }
+  }
+}

package/src/server/services/search/impls/exa/type.ts

@@ -0,0 +1,39 @@
+export interface ExaSearchParameters {
+  category?: string;
+  endCrawlDate?: string;
+  endPublishedDate?: string;
+  excludeDomains?: string[];
+  excludeText?: string[];
+  includeDomains?: string[];
+  includeText?: string[];
+  numResults?: number;
+  query: string;
+  startCrawlDate?: string;
+  startPublishedDate?: string;
+  type?: string;
+}
+
+interface ExaCostDollars {
+  total: number;
+}
+
+interface ExaResults {
+  author?: string | null;
+  favicon?: string;
+  id?: string;
+  image?: string;
+  publishedDate?: string | null;
+  score?: number | null;
+  summery?: string;
+  text: string;
+  title: string;
+  url: string;
+}
+
+export interface ExaResponse {
+  costDollars?: ExaCostDollars;
+  requestId?: string;
+  resolvedSearchType?: string;
+  results: ExaResults[];
+  searchType?: string;
+}

package/src/server/services/search/impls/firecrawl/index.ts

@@ -0,0 +1,128 @@
+import { TRPCError } from '@trpc/server';
+import debug from 'debug';
+import urlJoin from 'url-join';
+
+import { SearchParams, UniformSearchResponse, UniformSearchResult } from '@/types/tool/search';
+
+import { SearchServiceImpl } from '../type';
+import { FirecrawlSearchParameters, FirecrawlResponse } from './type';
+
+const log = debug('lobe-search:Firecrawl');
+
+const timeRangeMapping = {
+  day: 'qdr:d',
+  month: 'qdr:m',
+  week: 'qdr:w',
+  year: 'qdr:y',
+};
+
+/**
+ * Firecrawl implementation of the search service
+ * Primarily used for web crawling
+ */
+export class FirecrawlImpl implements SearchServiceImpl {
+  private get apiKey(): string | undefined {
+    return process.env.FIRECRAWL_API_KEY;
+  }
+
+  private get baseUrl(): string {
+    // Assuming the base URL is consistent with the crawl endpoint
+    return process.env.FIRECRAWL_URL || 'https://api.firecrawl.dev/v1';
+  }
+
+  async query(query: string, params: SearchParams = {}): Promise<UniformSearchResponse> {
+    log('Starting Firecrawl query with query: "%s", params: %o', query, params);
+    const endpoint = urlJoin(this.baseUrl, '/search');
+
+    const defaultQueryParams: FirecrawlSearchParameters = {
+      limit: 15,
+      query,
+      /*
+      scrapeOptions: {
+        formats: ["markdown"]
+      },
+      */
+    };
+
+    let body: FirecrawlSearchParameters = {
+      ...defaultQueryParams,
+      tbs:
+        params?.searchTimeRange && params.searchTimeRange !== 'anytime'
+          ? timeRangeMapping[params.searchTimeRange as keyof typeof timeRangeMapping] ?? undefined
+          : undefined,
+    };
+
+    log('Constructed request body: %o', body);
+
+    let response: Response;
+    const startAt = Date.now();
+    let costTime = 0;
+    try {
+      log('Sending request to endpoint: %s', endpoint);
+      response = await fetch(endpoint, {
+        body: JSON.stringify(body),
+        headers: {
+          'Authorization': this.apiKey ? `Bearer ${this.apiKey}` : '',
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+      });
+      log('Received response with status: %d', response.status);
+      costTime = Date.now() - startAt;
+    } catch (error) {
+      log.extend('error')('Firecrawl fetch error: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'SERVICE_UNAVAILABLE',
+        message: 'Failed to connect to Firecrawl.',
+      });
+    }
+
+    if (!response.ok) {
+      const errorBody = await response.text();
+      log.extend('error')(
+        `Firecrawl request failed with status ${response.status}: %s`,
+        errorBody.length > 200 ? `${errorBody.slice(0, 200)}...` : errorBody,
+      );
+      throw new TRPCError({
+        cause: errorBody,
+        code: 'SERVICE_UNAVAILABLE',
+        message: `Firecrawl request failed: ${response.statusText}`,
+      });
+    }
+
+    try {
+      const firecrawlResponse = (await response.json()) as FirecrawlResponse;
+
+      log('Parsed Firecrawl response: %o', firecrawlResponse);
+
+      const mappedResults = (firecrawlResponse.data || []).map(
+        (result): UniformSearchResult => ({
+          category: 'general', // Default category
+          content: result.description || '', // Prioritize content, fallback to snippet
+          engines: ['firecrawl'], // Use 'firecrawl' as the engine name
+          parsedUrl: result.url ? new URL(result.url).hostname : '', // Basic URL parsing
+          score: 1, // Default score to 1
+          title: result.title || '',
+          url: result.url,
+        }),
+      );
+
+      log('Mapped %d results to SearchResult format', mappedResults.length);
+
+      return {
+        costTime,
+        query: query,
+        resultNumbers: mappedResults.length,
+        results: mappedResults,
+      };
+    } catch (error) {
+      log.extend('error')('Error parsing Firecrawl response: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'INTERNAL_SERVER_ERROR',
+        message: 'Failed to parse Firecrawl response.',
+      });
+    }
+  }
+}

package/src/server/services/search/impls/firecrawl/type.ts

@@ -0,0 +1,35 @@
+interface FirecrawlScrapeOptions {
+  formats: string[];
+}
+
+export interface FirecrawlSearchParameters {
+  country?: string;
+  lang?: string;
+  limit?: number;
+  query: string;
+  scrapeOptions?: FirecrawlScrapeOptions;
+  tbs?: string;
+  timeout?: number;
+}
+
+interface FirecrawlMetadata {
+  description?: string;
+  sourceURL?: string;
+  statusCode?: number;
+  title: string;
+}
+
+interface FirecrawlData {
+  description?: string;
+  html?: string;
+  links?: string[];
+  markdown?: string;
+  metadata?: FirecrawlMetadata;
+  title?: string;
+  url: string;
+}
+
+export interface FirecrawlResponse {
+  data: FirecrawlData[];
+  success?: boolean;
+}

package/src/server/services/search/impls/index.ts

@@ -1,13 +1,24 @@
+import { BochaImpl } from './bocha';
+import { ExaImpl } from './exa';
+import { FirecrawlImpl } from './firecrawl';
+import { JinaImpl } from './jina';
 import { Search1APIImpl } from './search1api';
 import { SearXNGImpl } from './searxng';
+import { TavilyImpl } from './tavily';
+
 import { SearchServiceImpl } from './type';
 
 /**
  * Available search service implementations
  */
 export enum SearchImplType {
+  Bocha = 'bocha',
+  Exa = 'exa',
+  Firecrawl = 'firecrawl',
+  Jina = 'jina',
   SearXNG = 'searxng',
   Search1API = 'search1api',
+  Tavily = 'tavily',
 }
 
 /**
@@ -17,10 +28,30 @@ export const createSearchServiceImpl = (
   type: SearchImplType = SearchImplType.SearXNG,
 ): SearchServiceImpl => {
   switch (type) {
+    case SearchImplType.Bocha: {
+      return new BochaImpl();
+    }
+
+    case SearchImplType.Exa: {
+      return new ExaImpl();
+    }
+
+    case SearchImplType.Firecrawl: {
+      return new FirecrawlImpl();
+    }
+
+    case SearchImplType.Jina: {
+      return new JinaImpl();
+    }
+
     case SearchImplType.SearXNG: {
       return new SearXNGImpl();
     }
 
+    case SearchImplType.Tavily: {
+      return new TavilyImpl();
+    }
+
     default: {
       return new Search1APIImpl();
     }

package/src/server/services/search/impls/jina/index.ts

@@ -0,0 +1,109 @@
+import { TRPCError } from '@trpc/server';
+import debug from 'debug';
+import urlJoin from 'url-join';
+
+import { SearchParams, UniformSearchResponse, UniformSearchResult } from '@/types/tool/search';
+
+import { SearchServiceImpl } from '../type';
+import { JinaSearchParameters, JinaResponse } from './type';
+
+const log = debug('lobe-search:Jina');
+
+/**
+ * Jina implementation of the search service
+ * Primarily used for web crawling
+ */
+export class JinaImpl implements SearchServiceImpl {
+  private get apiKey(): string | undefined {
+    return process.env.JINA_READER_API_KEY || process.env.JINA_API_KEY;
+  }
+
+  private get baseUrl(): string {
+    // Assuming the base URL is consistent with the crawl endpoint
+    return 'https://s.jina.ai';
+  }
+
+  async query(query: string, params: SearchParams = {}): Promise<UniformSearchResponse> {
+    log('Starting Jina query with query: "%s", params: %o', query, params);
+    const endpoint = urlJoin(this.baseUrl, '/');
+
+    let body: JinaSearchParameters = {
+      q: query,
+    };
+
+    log('Constructed request body: %o', body);
+
+    let response: Response;
+    const startAt = Date.now();
+    let costTime = 0;
+    try {
+      log('Sending request to endpoint: %s', endpoint);
+      response = await fetch(endpoint, {
+        body: JSON.stringify(body),
+        headers: {
+          'Accept': 'application/json',
+          'Authorization': this.apiKey ? `Bearer ${this.apiKey}` : '',
+          'Content-Type': 'application/json',
+          'X-Respond-With': 'no-content',
+        },
+        method: 'POST',
+      });
+      log('Received response with status: %d', response.status);
+      costTime = Date.now() - startAt;
+    } catch (error) {
+      log.extend('error')('Jina fetch error: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'SERVICE_UNAVAILABLE',
+        message: 'Failed to connect to Jina.',
+      });
+    }
+
+    if (!response.ok) {
+      const errorBody = await response.text();
+      log.extend('error')(
+        `Jina request failed with status ${response.status}: %s`,
+        errorBody.length > 200 ? `${errorBody.slice(0, 200)}...` : errorBody,
+      );
+      throw new TRPCError({
+        cause: errorBody,
+        code: 'SERVICE_UNAVAILABLE',
+        message: `Jina request failed: ${response.statusText}`,
+      });
+    }
+
+    try {
+      const jinaResponse = (await response.json()) as JinaResponse;
+
+      log('Parsed Jina response: %o', jinaResponse);
+
+      const mappedResults = (jinaResponse.data || []).map(
+        (result): UniformSearchResult => ({
+          category: 'general', // Default category
+          content: result.description || '', // Prioritize content, fallback to snippet
+          engines: ['jina'], // Use 'jina' as the engine name
+          parsedUrl: result.url ? new URL(result.url).hostname : '', // Basic URL parsing
+          score: 1, // Default score to 1
+          title: result.title || '',
+          url: result.url,
+        }),
+      );
+
+      log('Mapped %d results to SearchResult format', mappedResults.length);
+
+      return {
+        costTime,
+        query: query,
+        resultNumbers: mappedResults.length,
+        results: mappedResults,
+      };
+    } catch (error) {
+      log.extend('error')('Error parsing Jina response: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'INTERNAL_SERVER_ERROR',
+        message: 'Failed to parse Jina response.',
+      });
+    }
+  }
+}

package/src/server/services/search/impls/jina/type.ts

@@ -0,0 +1,26 @@
+export interface JinaSearchParameters {
+  q: string;
+}
+
+interface JinaUsage {
+  tokens: number;
+}
+
+interface JinaMeta {
+  usage: JinaUsage;
+}
+
+interface JinaData {
+  content?: string;
+  description?: string;
+  title: string;
+  url: string;
+  usage?: JinaUsage;
+}
+
+export interface JinaResponse {
+  code?: number;
+  data: JinaData[];
+  meta?: JinaMeta;
+  status?: number;
+}

package/src/server/services/search/impls/tavily/index.ts

@@ -0,0 +1,124 @@
+import { TRPCError } from '@trpc/server';
+import debug from 'debug';
+import urlJoin from 'url-join';
+
+import { SearchParams, UniformSearchResponse, UniformSearchResult } from '@/types/tool/search';
+
+import { SearchServiceImpl } from '../type';
+import { TavilySearchParameters, TavilyResponse } from './type';
+
+const log = debug('lobe-search:Tavily');
+
+/**
+ * Tavily implementation of the search service
+ * Primarily used for web crawling
+ */
+export class TavilyImpl implements SearchServiceImpl {
+  private get apiKey(): string | undefined {
+    return process.env.TAVILY_API_KEY;
+  }
+
+  private get baseUrl(): string {
+    // Assuming the base URL is consistent with the crawl endpoint
+    return 'https://api.tavily.com';
+  }
+
+  async query(query: string, params: SearchParams = {}): Promise<UniformSearchResponse> {
+    log('Starting Tavily query with query: "%s", params: %o', query, params);
+    const endpoint = urlJoin(this.baseUrl, '/search');
+
+    const defaultQueryParams: TavilySearchParameters = {
+      include_answer: false,
+      include_image_descriptions: true,
+      include_images: false,
+      include_raw_content: false,
+      max_results: 15,
+      query,
+      search_depth: process.env.TAVILY_SEARCH_DEPTH || 'basic' // basic or advanced
+    };
+
+    let body: TavilySearchParameters = {
+      ...defaultQueryParams,
+      time_range:
+        params?.searchTimeRange && params.searchTimeRange !== 'anytime'
+          ? params.searchTimeRange
+          : undefined,
+      topic:
+        // Tavily 只支持 news 和 general 两种类型
+        params?.searchCategories?.filter(cat => ['news', 'general'].includes(cat))?.[0],
+    };
+
+    log('Constructed request body: %o', body);
+
+    let response: Response;
+    const startAt = Date.now();
+    let costTime = 0;
+    try {
+      log('Sending request to endpoint: %s', endpoint);
+      response = await fetch(endpoint, {
+        body: JSON.stringify(body),
+        headers: {
+          'Authorization': this.apiKey ? `Bearer ${this.apiKey}` : '',
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+      });
+      log('Received response with status: %d', response.status);
+      costTime = Date.now() - startAt;
+    } catch (error) {
+      log.extend('error')('Tavily fetch error: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'SERVICE_UNAVAILABLE',
+        message: 'Failed to connect to Tavily.',
+      });
+    }
+
+    if (!response.ok) {
+      const errorBody = await response.text();
+      log.extend('error')(
+        `Tavily request failed with status ${response.status}: %s`,
+        errorBody.length > 200 ? `${errorBody.slice(0, 200)}...` : errorBody,
+      );
+      throw new TRPCError({
+        cause: errorBody,
+        code: 'SERVICE_UNAVAILABLE',
+        message: `Tavily request failed: ${response.statusText}`,
+      });
+    }
+
+    try {
+      const tavilyResponse = (await response.json()) as TavilyResponse;
+
+      log('Parsed Tavily response: %o', tavilyResponse);
+
+      const mappedResults = (tavilyResponse.results || []).map(
+        (result): UniformSearchResult => ({
+          category: body.topic || 'general', // Default category
+          content: result.content || '', // Prioritize content, fallback to snippet
+          engines: ['tavily'], // Use 'tavily' as the engine name
+          parsedUrl: result.url ? new URL(result.url).hostname : '', // Basic URL parsing
+          score: result.score || 0, // Default score to 0 if undefined
+          title: result.title || '',
+          url: result.url,
+        }),
+      );
+
+      log('Mapped %d results to SearchResult format', mappedResults.length);
+
+      return {
+        costTime,
+        query: query,
+        resultNumbers: mappedResults.length,
+        results: mappedResults,
+      };
+    } catch (error) {
+      log.extend('error')('Error parsing Tavily response: %o', error);
+      throw new TRPCError({
+        cause: error,
+        code: 'INTERNAL_SERVER_ERROR',
+        message: 'Failed to parse Tavily response.',
+      });
+    }
+  }
+}

package/src/server/services/search/impls/tavily/type.ts

@@ -0,0 +1,36 @@
+export interface TavilySearchParameters {
+  chunks_per_source?: number;
+  days?: number;
+  exclude_domains?: string[];
+  include_answer?: boolean | string;
+  include_domains?: string[];
+  include_image_descriptions?: boolean;
+  include_images?: boolean;
+  include_raw_content?: boolean;
+  max_results?: number;
+  query: string;
+  search_depth?: string;
+  time_range?: string;
+  topic?: string;
+}
+
+interface TavilyImages {
+  description?: string;
+  url: string;
+}
+
+interface TavilyResults {
+  content?: string;
+  raw_content?: string | null;
+  score?: number;
+  title?: string;
+  url: string;
+}
+
+export interface TavilyResponse {
+  answer?: string;
+  images?: TavilyImages[];
+  query: string;
+  response_time: number;
+  results: TavilyResults[];
+}