@lobehub/chat 1.90.4 → 1.91.0

package/CHANGELOG.md

@@ -2,6 +2,39 @@
 
 # Changelog
 
+## [Version 1.91.0](https://github.com/lobehub/lobe-chat/compare/v1.90.4...v1.91.0)
+
+<sup>Released on **2025-06-03**</sup>
+
+#### ✨ Features
+
+- **misc**: Add more provider support for search & crawl.
+
+#### 💄 Styles
+
+- **misc**: Update modelscope models.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's improved
+
+- **misc**: Add more provider support for search & crawl, closes [#8033](https://github.com/lobehub/lobe-chat/issues/8033) ([23fade3](https://github.com/lobehub/lobe-chat/commit/23fade3))
+
+#### Styles
+
+- **misc**: Update modelscope models, closes [#8057](https://github.com/lobehub/lobe-chat/issues/8057) ([3e02c25](https://github.com/lobehub/lobe-chat/commit/3e02c25))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.90.4](https://github.com/lobehub/lobe-chat/compare/v1.90.3...v1.90.4)
 
 <sup>Released on **2025-06-02**</sup>

package/changelog/v1.json

@@ -1,4 +1,16 @@
 [
+  {
+    "children": {
+      "features": [
+        "Add more provider support for search & crawl."
+      ],
+      "improvements": [
+        "Update modelscope models."
+      ]
+    },
+    "date": "2025-06-03",
+    "version": "1.91.0"
+  },
   {
     "children": {
       "improvements": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.90.4",
+  "version": "1.91.0",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/packages/web-crawler/src/crawImpl/exa.ts

@@ -0,0 +1,93 @@
+import { CrawlImpl, CrawlSuccessResult } from '../type';
+import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../utils/errorType';
+import { DEFAULT_TIMEOUT, withTimeout } from '../utils/withTimeout';
+
+interface ExaResults {
+  author?: string;
+  favicon?: string;
+  id?: string;
+  image?: string;
+  publishedDate?: string;
+  summary?: string;
+  text: string;
+  title: string;
+  url: string;
+}
+
+interface ExaResponse {
+  requestId?: string;
+  results: ExaResults[];
+}
+
+export const exa: CrawlImpl = async (url) => {
+  // Get API key from environment variable
+  const apiKey = process.env.EXA_API_KEY;
+
+  let res: Response;
+
+  try {
+    res = await withTimeout(
+      fetch('https://api.exa.ai/contents', {
+        body: JSON.stringify({
+          livecrawl: 'fallback', // always, fallback
+          text: true,
+          urls: [url],
+        }),
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': !apiKey ? '' : apiKey,
+        },
+        method: 'POST',
+      }),
+      DEFAULT_TIMEOUT,
+    );
+  } catch (e) {
+    const error = e as Error;
+    if (error.message === 'fetch failed') {
+      throw new NetworkConnectionError();
+    }
+
+    if (error instanceof TimeoutError) {
+      throw error;
+    }
+
+    throw e;
+  }
+
+  if (!res.ok) {
+    if (res.status === 404) {
+      throw new PageNotFoundError(res.statusText);
+    }
+
+    throw new Error(`Exa request failed with status ${res.status}: ${res.statusText}`);
+  }
+
+  try {
+    const data = (await res.json()) as ExaResponse;
+
+    if (!data.results || data.results.length === 0) {
+      console.warn( 'Exa API returned no results for URL:', url )
+      return
+    }
+
+    const firstResult = data.results[0];
+
+    // Check if content is empty or too short
+    if (!firstResult.text || firstResult.text.length < 100) {
+      return;
+    }
+
+    return {
+      content: firstResult.text,
+      contentType: 'text',
+      length: firstResult.text.length,
+      siteName: new URL(url).hostname,
+      title: firstResult.title,
+      url: firstResult.url || url,
+    } satisfies CrawlSuccessResult;
+  } catch (error) {
+    console.error(error);
+  }
+
+  return;
+};

package/packages/web-crawler/src/crawImpl/firecrawl.ts

@@ -0,0 +1,97 @@
+import { CrawlImpl, CrawlSuccessResult } from '../type';
+import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../utils/errorType';
+import { DEFAULT_TIMEOUT, withTimeout } from '../utils/withTimeout';
+
+interface FirecrawlMetadata {
+  description: string;
+  keywords: string;
+  language: string;
+  ogDescription?: string;
+  ogImage?: string;
+  ogLocaleAlternate?: string[];
+  ogSiteName?: string;
+  ogTitle?: string;
+  ogUrl?: string;
+  robots: string;
+  statusCode: number;
+  sourceURL: string;
+  title: string;
+}
+
+interface FirecrawlResults {
+  html?: string;
+  markdown?: string;
+  metadata: FirecrawlMetadata;
+}
+
+interface FirecrawlResponse {
+  success: boolean;
+  data: FirecrawlResults;
+}
+
+export const firecrawl: CrawlImpl = async (url) => {
+  // Get API key from environment variable
+  const apiKey = process.env.FIRECRAWL_API_KEY;
+  const baseUrl = process.env.FIRECRAWL_URL || 'https://api.firecrawl.dev/v1';
+
+  let res: Response;
+
+  try {
+    res = await withTimeout(
+      fetch(`${baseUrl}/scrape`, {
+        body: JSON.stringify({
+          formats: ["markdown"], // ["markdown", "html"]
+          url,
+        }),
+        headers: {
+          'Authorization': !apiKey ? '' : `Bearer ${apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+      }),
+      DEFAULT_TIMEOUT,
+    );
+  } catch (e) {
+    const error = e as Error;
+    if (error.message === 'fetch failed') {
+      throw new NetworkConnectionError();
+    }
+
+    if (error instanceof TimeoutError) {
+      throw error;
+    }
+
+    throw e;
+  }
+
+  if (!res.ok) {
+    if (res.status === 404) {
+      throw new PageNotFoundError(res.statusText);
+    }
+
+    throw new Error(`Firecrawl request failed with status ${res.status}: ${res.statusText}`);
+  }
+
+  try {
+    const data = (await res.json()) as FirecrawlResponse;
+
+    // Check if content is empty or too short
+    if (!data.data.markdown || data.data.markdown.length < 100) {
+      return;
+    }
+
+    return {
+      content: data.data.markdown,
+      contentType: 'text',
+      description: data.data.metadata.description,
+      length: data.data.markdown.length,
+      siteName: new URL(url).hostname,
+      title: data.data.metadata.title,
+      url: url,
+    } satisfies CrawlSuccessResult;
+  } catch (error) {
+    console.error(error);
+  }
+
+  return;
+};

package/packages/web-crawler/src/crawImpl/index.ts

@@ -1,13 +1,19 @@
 import { browserless } from './browserless';
+import { exa } from './exa';
+import { firecrawl } from './firecrawl';
 import { jina } from './jina';
 import { naive } from './naive';
 import { search1api } from './search1api';
+import { tavily } from './tavily';
 
 export const crawlImpls = {
   browserless,
+  exa,
+  firecrawl,
   jina,
   naive,
   search1api,
+  tavily,
 };
 
 export type CrawlImplType = keyof typeof crawlImpls;

package/packages/web-crawler/src/crawImpl/tavily.ts

@@ -0,0 +1,94 @@
+import { CrawlImpl, CrawlSuccessResult } from '../type';
+import { NetworkConnectionError, PageNotFoundError, TimeoutError } from '../utils/errorType';
+import { DEFAULT_TIMEOUT, withTimeout } from '../utils/withTimeout';
+
+interface TavilyResults {
+  images?: string[];
+  raw_content: string;
+  url: string;
+}
+
+interface TavilyFailedResults {
+  error?: string;
+  url: string;
+}
+
+interface TavilyResponse {
+  base_url: string;
+  failed_results?: TavilyFailedResults[];
+  response_time: number;
+  results: TavilyResults[];
+}
+
+export const tavily: CrawlImpl = async (url) => {
+  // Get API key from environment variable
+  const apiKey = process.env.TAVILY_API_KEY;
+
+  let res: Response;
+
+  try {
+    res = await withTimeout(
+      fetch('https://api.tavily.com/extract', {
+        body: JSON.stringify({
+          extract_depth: process.env.TAVILY_EXTRACT_DEPTH || 'basic', // basic or advanced
+          include_images: false,
+          urls: url,
+        }),
+        headers: {
+          'Authorization': !apiKey ? '' : `Bearer ${apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+      }),
+      DEFAULT_TIMEOUT,
+    );
+  } catch (e) {
+    const error = e as Error;
+    if (error.message === 'fetch failed') {
+      throw new NetworkConnectionError();
+    }
+
+    if (error instanceof TimeoutError) {
+      throw error;
+    }
+
+    throw e;
+  }
+
+  if (!res.ok) {
+    if (res.status === 404) {
+      throw new PageNotFoundError(res.statusText);
+    }
+
+    throw new Error(`Tavily request failed with status ${res.status}: ${res.statusText}`);
+  }
+
+  try {
+    const data = (await res.json()) as TavilyResponse;
+
+    if (!data.results || data.results.length === 0) {
+      console.warn( 'Tavily API returned no results for URL:', url )
+      return
+    }
+
+    const firstResult = data.results[0];
+
+    // Check if content is empty or too short
+    if (!firstResult.raw_content || firstResult.raw_content.length < 100) {
+      return;
+    }
+
+    return {
+      content: firstResult.raw_content,
+      contentType: 'text',
+      length: firstResult.raw_content.length,
+      siteName: new URL(url).hostname,
+      title: new URL(url).hostname,
+      url: firstResult.url || url,
+    } satisfies CrawlSuccessResult;
+  } catch (error) {
+    console.error(error);
+  }
+
+  return;
+};

package/src/config/aiModels/modelscope.ts

@@ -6,10 +6,10 @@ const modelscopeChatModels: AIChatModelCard[] = [
       functionCall: true,
     },
     contextWindowTokens: 131_072,
-    description: 'DeepSeek-V3是DeepSeek第三代模型，在多项基准测试中表现优异。',
-    displayName: 'DeepSeek-V3-0324',
+    description: 'DeepSeek R1 通过利用增加的计算资源和在后训练过程中引入算法优化机制，显著提高了其推理和推断能力的深度。该模型在各种基准评估中表现出色，包括数学、编程和一般逻辑方面。其整体性能现已接近领先模型，如 O3 和 Gemini 2.5 Pro。',
+    displayName: 'DeepSeek-R1-0528',
     enabled: true,
-    id: 'deepseek-ai/DeepSeek-V3-0324',
+    id: 'deepseek-ai/DeepSeek-R1-0528',
     type: 'chat',
   },
   {

package/src/config/modelProviders/modelscope.ts

@@ -5,11 +5,11 @@ const ModelScope: ModelProviderCard = {
   chatModels: [
     {
       contextWindowTokens: 131_072,
-      description: 'DeepSeek-V3是DeepSeek第三代模型，在多项基准测试中表现优异。',
-      displayName: 'DeepSeek-V3-0324',
+      description: 'DeepSeek R1 通过利用增加的计算资源和在后训练过程中引入算法优化机制，显著提高了其推理和推断能力的深度。该模型在各种基准评估中表现出色，包括数学、编程和一般逻辑方面。其整体性能现已接近领先模型，如 O3 和 Gemini 2.5 Pro。',
+      displayName: 'DeepSeek-R1-0528',
       enabled: true,
       functionCall: true,
-      id: 'deepseek-ai/DeepSeek-V3-0324',
+      id: 'deepseek-ai/DeepSeek-R1-0528',
     },
     {
       contextWindowTokens: 131_072,

package/src/database/client/migrations.json

@@ -479,5 +479,15 @@
     "bps": true,
     "folderMillis": 1746724476380,
     "hash": "0518cd9882f7ea38eb498b31c8dda73fb56bbc3aa55445ecbc7a9e716631d047"
+  },
+  {
+    "sql": [
+      "-- Custom SQL migration file, put your code below! --\nUPDATE agents SET chat_config = jsonb_set(chat_config, '{enableReasoningEffort}', 'false') WHERE chat_config ->> 'enableReasoningEffort' = 'true';\n",
+      "\nUPDATE agents SET params = params - 'reasoning_effort' WHERE params ? 'reasoning_effort';\n",
+      "\nDELETE FROM ai_providers WHERE id = 'doubao';"
+    ],
+    "bps": true,
+    "folderMillis": 1748925630721,
+    "hash": "15815e06e3b188933119f380ae12ef8694f2c5f8003cebb397ed00053a065be4"
   }
 ]

package/src/database/migrations/0023_remove_param_and_doubao.sql

@@ -0,0 +1,6 @@
+-- Custom SQL migration file, put your code below! --
+UPDATE agents SET chat_config = jsonb_set(chat_config, '{enableReasoningEffort}', 'false') WHERE chat_config ->> 'enableReasoningEffort' = 'true';
+--> statement-breakpoint
+UPDATE agents SET params = params - 'reasoning_effort' WHERE params ? 'reasoning_effort';
+--> statement-breakpoint
+DELETE FROM ai_providers WHERE id = 'doubao';