@lobehub/chat 1.66.5 → 1.67.0

package/packages/web-crawler/src/utils/htmlToMarkdown.test.ts

@@ -0,0 +1,35 @@
+import { readFileSync } from 'node:fs';
+import * as path from 'node:path';
+import { expect } from 'vitest';
+
+import { FilterOptions } from '../type';
+import { htmlToMarkdown } from './htmlToMarkdown';
+
+interface TestItem {
+  file: string;
+  url: string;
+  filterOptions?: FilterOptions;
+}
+const list: TestItem[] = [
+  {
+    file: 'terms.html',
+    url: 'https://lobehub.com/terms',
+  },
+  {
+    file: 'yingchao.html',
+    url: 'https://www.qiumiwu.com/standings/yingchao',
+    filterOptions: { pureText: true, enableReadability: false },
+  },
+];
+
+describe('htmlToMarkdown', () => {
+  list.forEach((item) => {
+    it(`should transform ${item.file} to markdown`, () => {
+      const html = readFileSync(path.join(__dirname, `./html/${item.file}`), { encoding: 'utf8' });
+
+      const data = htmlToMarkdown(html, { url: item.url, filterOptions: item.filterOptions || {} });
+
+      expect(data).toMatchSnapshot();
+    });
+  });
+});

package/packages/web-crawler/src/utils/htmlToMarkdown.ts

@@ -0,0 +1,45 @@
+import { Readability } from '@mozilla/readability';
+import { Window } from 'happy-dom';
+import { NodeHtmlMarkdown, type TranslatorConfigObject } from 'node-html-markdown';
+
+import { FilterOptions } from '../type';
+
+export const htmlToMarkdown = (
+  html: string,
+  { url, filterOptions }: { filterOptions: FilterOptions; url: string },
+) => {
+  const window = new Window({ url });
+
+  const document = window.document;
+  document.body.innerHTML = html;
+
+  // @ts-expect-error reason: Readability expects a Document type
+  const parsedContent = new Readability(document).parse();
+
+  const useReadability = filterOptions.enableReadability ?? true;
+
+  let htmlNode = html;
+
+  if (useReadability && parsedContent?.content) {
+    htmlNode = parsedContent?.content;
+  }
+
+  const customTranslators = (
+    filterOptions.pureText
+      ? {
+          a: {
+            postprocess: (_: string, content: string) => content,
+          },
+          img: {
+            ignore: true,
+          },
+        }
+      : {}
+  ) as TranslatorConfigObject;
+
+  const nodeHtmlMarkdown = new NodeHtmlMarkdown({}, customTranslators);
+
+  const content = nodeHtmlMarkdown.translate(htmlNode);
+
+  return { ...parsedContent, content };
+};

package/packages/web-crawler/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "module": "CommonJS",
+    "target": "ESNext",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "sourceMap": true,
+    "skipDefaultLibCheck": true,
+    "jsx": "react-jsx",
+    "baseUrl": ".",
+    "allowSyntheticDefaultImports": true,
+    "moduleResolution": "node",
+    "forceConsistentCasingInFileNames": true,
+    "noImplicitReturns": true,
+    "noUnusedLocals": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "types": ["vitest/globals"]
+  }
+}

package/pnpm-workspace.yaml

@@ -0,0 +1,3 @@
+packages:
+  - 'packages/**'
+  - '.'

package/src/config/aiModels/openai.ts

@@ -21,6 +21,7 @@ export const openaiChatModels: AIChatModelCard[] = [
     id: 'o3-mini',
     maxOutput: 100_000,
     pricing: {
+      cachedInput: 0.55,
       input: 1.1,
       output: 4.4,
     },
@@ -39,6 +40,7 @@ export const openaiChatModels: AIChatModelCard[] = [
     id: 'o1-mini',
     maxOutput: 65_536,
     pricing: {
+      cachedInput: 0.55,
       input: 1.1,
       output: 4.4,
     },
@@ -58,6 +60,7 @@ export const openaiChatModels: AIChatModelCard[] = [
     id: 'o1',
     maxOutput: 100_000,
     pricing: {
+      cachedInput: 7.5,
       input: 15,
       output: 60,
     },
@@ -82,6 +85,26 @@ export const openaiChatModels: AIChatModelCard[] = [
     releasedAt: '2024-09-12',
     type: 'chat',
   },
+  {
+    abilities: {
+      functionCall: true,
+      vision: true,
+    },
+    contextWindowTokens: 128_000,
+    description:
+      'GPT-4.5 的研究预览版，它是我们迄今为止最大、最强大的 GPT 模型。它拥有广泛的世界知识，并能更好地理解用户意图，使其在创造性任务和自主规划方面表现出色。GPT-4.5 可接受文本和图像输入，并生成文本输出（包括结构化输出）。支持关键的开发者功能，如函数调用、批量 API 和流式输出。在需要创造性、开放式思考和对话的任务（如写作、学习或探索新想法）中，GPT-4.5 表现尤为出色。知识截止日期为 2023 年 10 月。',
+    displayName: 'GPT-4.5 Preview',
+    enabled: true,
+    id: 'gpt-4.5-preview',
+    maxOutput: 16_384,
+    pricing: {
+      cachedInput: 37.5,
+      input: 75,
+      output: 150,
+    },
+    releasedAt: '2025-02-27',
+    type: 'chat',
+  },
   {
     abilities: {
       functionCall: true,
@@ -93,8 +116,9 @@ export const openaiChatModels: AIChatModelCard[] = [
     displayName: 'GPT-4o mini',
     enabled: true,
     id: 'gpt-4o-mini',
-    maxOutput: 16_385,
+    maxOutput: 16_384,
     pricing: {
+      cachedInput: 0.075,
       input: 0.15,
       output: 0.6,
     },
@@ -110,7 +134,6 @@ export const openaiChatModels: AIChatModelCard[] = [
     description:
       'ChatGPT-4o 是一款动态模型，实时更新以保持当前最新版本。它结合了强大的语言理解与生成能力，适合于大规模应用场景，包括客户服务、教育和技术支持。',
     displayName: 'GPT-4o 1120',
-    enabled: true,
     id: 'gpt-4o-2024-11-20',
     pricing: {
       input: 2.5,
@@ -131,6 +154,7 @@ export const openaiChatModels: AIChatModelCard[] = [
     enabled: true,
     id: 'gpt-4o',
     pricing: {
+      cachedInput: 1.25,
       input: 2.5,
       output: 10,
     },
@@ -350,7 +374,7 @@ export const openaiChatModels: AIChatModelCard[] = [
     abilities: {
       functionCall: true,
     },
-    contextWindowTokens: 16_385,
+    contextWindowTokens: 16_384,
     description:
       'GPT 3.5 Turbo，适用于各种文本生成和理解任务，Currently points to gpt-3.5-turbo-0125',
     displayName: 'GPT-3.5 Turbo',
@@ -365,7 +389,7 @@ export const openaiChatModels: AIChatModelCard[] = [
     abilities: {
       functionCall: true,
     },
-    contextWindowTokens: 16_385,
+    contextWindowTokens: 16_384,
     description:
       'GPT 3.5 Turbo，适用于各种文本生成和理解任务，Currently points to gpt-3.5-turbo-0125',
     displayName: 'GPT-3.5 Turbo 0125',
@@ -381,7 +405,7 @@ export const openaiChatModels: AIChatModelCard[] = [
     abilities: {
       functionCall: true,
     },
-    contextWindowTokens: 16_385,
+    contextWindowTokens: 16_384,
     description:
       'GPT 3.5 Turbo，适用于各种文本生成和理解任务，Currently points to gpt-3.5-turbo-0125',
     displayName: 'GPT-3.5 Turbo 1106',

package/src/database/server/models/__tests__/message.test.ts

@@ -1271,7 +1271,7 @@ describe('MessageModel', () => {
       const result = await messageModel.getHeatmaps();
 
       // 断言结果
-      expect(result.length).toBeGreaterThan(366);
+      expect(result.length).toBeGreaterThanOrEqual(366);
       expect(result.length).toBeLessThan(368);
 
       // 检查两天前的数据
@@ -1389,7 +1389,7 @@ describe('MessageModel', () => {
       const result = await messageModel.getHeatmaps();
 
       // 断言结果
-      expect(result.length).toBeGreaterThan(366);
+      expect(result.length).toBeGreaterThanOrEqual(366);
       expect(result.length).toBeLessThan(368);
 
       // 检查所有数据的 count 和 level 是否为 0

package/src/features/Conversation/Messages/Assistant/Tool/Render/CustomRender.tsx

@@ -1,14 +1,9 @@
-import { Icon } from '@lobehub/ui';
-import { ConfigProvider, Empty } from 'antd';
-import { useTheme } from 'antd-style';
-import { LucideSquareArrowLeft, LucideSquareArrowRight } from 'lucide-react';
-import { memo, useContext, useEffect } from 'react';
-import { useTranslation } from 'react-i18next';
-import { Center, Flexbox } from 'react-layout-kit';
+import { memo, useEffect } from 'react';
+import { Flexbox } from 'react-layout-kit';
 
 import PluginRender from '@/features/PluginsUI/Render';
 import { useChatStore } from '@/store/chat';
-import { chatPortalSelectors, chatSelectors } from '@/store/chat/selectors';
+import { chatSelectors } from '@/store/chat/selectors';
 import { ChatMessage } from '@/types/message';
 
 import Arguments from './Arguments';
@@ -30,14 +25,7 @@ const CustomRender = memo<CustomRenderProps>(
     setShowPluginRender,
     pluginError,
   }) => {
-    const [loading, isMessageToolUIOpen] = useChatStore((s) => [
-      chatSelectors.isPluginApiInvoking(id)(s),
-      chatPortalSelectors.isPluginUIOpen(id)(s),
-    ]);
-    const { direction } = useContext(ConfigProvider.ConfigContext);
-    const { t } = useTranslation('plugin');
-
-    const theme = useTheme();
+    const [loading] = useChatStore((s) => [chatSelectors.isPluginApiInvoking(id)(s)]);
 
     useEffect(() => {
       if (!plugin?.type || loading) return;
@@ -45,25 +33,6 @@ const CustomRender = memo<CustomRenderProps>(
       setShowPluginRender(plugin?.type !== 'default');
     }, [plugin?.type, loading]);
 
-    if (isMessageToolUIOpen)
-      return (
-        <Center paddingBlock={8} style={{ background: theme.colorFillQuaternary, borderRadius: 4 }}>
-          <Empty
-            description={t('showInPortal')}
-            image={
-              <Icon
-                color={theme.colorTextQuaternary}
-                icon={direction === 'rtl' ? LucideSquareArrowLeft : LucideSquareArrowRight}
-                size={'large'}
-              />
-            }
-            styles={{
-              image: { height: 24 },
-            }}
-          />
-        </Center>
-      );
-
     if (loading) return <Arguments arguments={requestArgs} shine />;
 
     return (

package/src/features/Conversation/Messages/Assistant/Tool/Render/index.tsx

@@ -1,12 +1,12 @@
 import { Suspense, memo } from 'react';
 
 import { LOADING_FLAT } from '@/const/message';
-import ErrorResponse from '@/features/Conversation/Messages/Assistant/Tool/Render/ErrorResponse';
 import { useChatStore } from '@/store/chat';
 import { chatSelectors } from '@/store/chat/selectors';
 
 import Arguments from './Arguments';
 import CustomRender from './CustomRender';
+import ErrorResponse from './ErrorResponse';
 
 interface RenderProps {
   messageId: string;

package/src/features/PluginsUI/Render/BuiltinType/index.tsx

@@ -7,6 +7,7 @@ import Loading from '../Loading';
 import { useParseContent } from '../useParseContent';
 
 export interface BuiltinTypeProps {
+  apiName?: string;
   arguments?: string;
   content: string;
   id: string;
@@ -25,6 +26,7 @@ const BuiltinType = memo<BuiltinTypeProps>(
     identifier,
     loading,
     pluginError,
+    apiName,
   }) => {
     const { isJSON, data } = useParseContent(content);
 
@@ -40,6 +42,7 @@ const BuiltinType = memo<BuiltinTypeProps>(
 
     return (
       <Render
+        apiName={apiName}
         args={args}
         content={data}
         identifier={identifier}

package/src/features/PluginsUI/Render/index.tsx

@@ -40,6 +40,7 @@ const PluginRender = memo<PluginRenderProps>(
       case 'builtin': {
         return (
           <BuiltinType
+            apiName={payload?.apiName}
             arguments={argumentsStr}
             content={content}
             id={id}

package/src/features/Portal/Plugins/Body/ToolRender.tsx

@@ -40,6 +40,7 @@ const ToolRender = memo(() => {
 
   return (
     <Render
+      apiName={plugin.apiName}
       arguments={args}
       identifier={plugin.identifier}
       messageId={messageId}

package/src/locales/default/plugin.ts

@@ -141,6 +141,18 @@ export default {
       close: '删除',
       confirm: '已完成配置并重试',
     },
+    crawPages: {
+      crawling: '链接识别中',
+      detail: {
+        preview: '预览',
+        raw: '原始文本',
+        tooLong: '文本内容过长，对话上下文仅保留前 10000 字符，超过部分不计入会话上下文',
+      },
+      meta: {
+        crawler: '抓取模式',
+        words: '字符数',
+      },
+    },
     searchxng: {
       baseURL: '请输入',
       description: '请输入 SearchXNG 的网址，即可开始联网搜索',

package/src/server/routers/tools/search.ts

@@ -1,4 +1,6 @@
+import { Crawler } from '@lobechat/web-crawler';
 import { TRPCError } from '@trpc/server';
+import pMap from 'p-map';
 import { z } from 'zod';
 
 import { toolsEnv } from '@/config/tools';
@@ -10,6 +12,27 @@ import { SEARCH_SEARXNG_NOT_CONFIG } from '@/types/tool/search';
 const searchProcedure = isServerMode ? authedProcedure : passwordProcedure;
 
 export const searchRouter = router({
+  crawlPages: searchProcedure
+    .input(
+      z.object({
+        impls: z.string().array().optional(),
+        urls: z.string().array(),
+      }),
+    )
+    .mutation(async ({ input }) => {
+      const crawler = new Crawler();
+
+      const results = await pMap(
+        input.urls,
+        async (url) => {
+          return await crawler.crawl({ impls: input.impls, url });
+        },
+        { concurrency: 10 },
+      );
+
+      return { results };
+    }),
+
   query: searchProcedure
     .input(
       z.object({

package/src/services/search.ts

@@ -4,6 +4,14 @@ class SearchService {
   search(query: string, searchEngine?: string[]) {
     return toolsClient.search.query.query({ query, searchEngine });
   }
+
+  crawlPage(url: string) {
+    return toolsClient.search.crawlPages.mutate({ urls: [url] });
+  }
+
+  crawlPages(urls: string[]) {
+    return toolsClient.search.crawlPages.mutate({ urls });
+  }
 }
 
 export const searchService = new SearchService();

package/src/store/chat/slices/builtinTool/actions/searXNG.ts

@@ -13,6 +13,16 @@ import {
 import { nanoid } from '@/utils/uuid';
 
 export interface SearchAction {
+  crawlMultiPages: (
+    id: string,
+    params: { urls: string[] },
+    aiSummary?: boolean,
+  ) => Promise<boolean | undefined>;
+  crawlSinglePage: (
+    id: string,
+    params: { url: string },
+    aiSummary?: boolean,
+  ) => Promise<boolean | undefined>;
   /**
    * 重新发起搜索
    * @description 会更新插件的 arguments 参数，然后再次搜索
@@ -28,6 +38,7 @@ export interface SearchAction {
     data: SearchQuery,
     aiSummary?: boolean,
   ) => Promise<void | boolean>;
+  togglePageContent: (url: string) => void;
   toggleSearchLoading: (id: string, loading: boolean) => void;
 }
 
@@ -37,12 +48,47 @@ export const searchSlice: StateCreator<
   [],
   SearchAction
 > = (set, get) => ({
+  crawlMultiPages: async (id, params, aiSummary = true) => {
+    const { internal_updateMessageContent } = get();
+    get().toggleSearchLoading(id, true);
+    const response = await searchService.crawlPages(params.urls);
+
+    await get().updatePluginState(id, response);
+    get().toggleSearchLoading(id, false);
+    const { results } = response;
+
+    if (!results) return;
+
+    const content = results.map((item) =>
+      'errorMessage' in item
+        ? item
+        : {
+            ...item.data,
+            // if crawl too many content
+            // slice the top 10000 char
+            content: item.data.content?.slice(0, 10_000),
+          },
+    );
+
+    await internal_updateMessageContent(id, JSON.stringify(content));
+
+    // if aiSummary is true, then trigger ai message
+    return aiSummary;
+  },
+
+  crawlSinglePage: async (id, params, aiSummary) => {
+    const { crawlMultiPages } = get();
+
+    return await crawlMultiPages(id, { urls: [params.url] }, aiSummary);
+  },
+
   reSearchWithSearXNG: async (id, data, options) => {
     get().toggleSearchLoading(id, true);
     await get().updatePluginArguments(id, data);
 
     await get().searchWithSearXNG(id, data, options?.aiSummary);
   },
+
   saveSearXNGSearchResult: async (id) => {
     const message = chatSelectors.getMessageById(id)(get());
     if (!message || !message.plugin) return;
@@ -133,6 +179,10 @@ export const searchSlice: StateCreator<
     return aiSummary;
   },
 
+  togglePageContent: (url) => {
+    set({ activePageContentUrl: url });
+  },
+
   toggleSearchLoading: (id, loading) => {
     set(
       { searchLoading: { ...get().searchLoading, [id]: loading } },

package/src/store/chat/slices/builtinTool/initialState.ts

@@ -1,6 +1,7 @@
 import { FileItem } from '@/types/files';
 
 export interface ChatToolState {
+  activePageContentUrl?: string;
   dalleImageLoading: Record<string, boolean>;
   dalleImageMap: Record<string, FileItem>;
   searchLoading: Record<string, boolean>;