@lobehub/chat 1.66.6 → 1.67.0

package/packages/web-crawler/src/utils/htmlToMarkdown.test.ts

@@ -0,0 +1,35 @@
+import { readFileSync } from 'node:fs';
+import * as path from 'node:path';
+import { expect } from 'vitest';
+
+import { FilterOptions } from '../type';
+import { htmlToMarkdown } from './htmlToMarkdown';
+
+interface TestItem {
+  file: string;
+  url: string;
+  filterOptions?: FilterOptions;
+}
+const list: TestItem[] = [
+  {
+    file: 'terms.html',
+    url: 'https://lobehub.com/terms',
+  },
+  {
+    file: 'yingchao.html',
+    url: 'https://www.qiumiwu.com/standings/yingchao',
+    filterOptions: { pureText: true, enableReadability: false },
+  },
+];
+
+describe('htmlToMarkdown', () => {
+  list.forEach((item) => {
+    it(`should transform ${item.file} to markdown`, () => {
+      const html = readFileSync(path.join(__dirname, `./html/${item.file}`), { encoding: 'utf8' });
+
+      const data = htmlToMarkdown(html, { url: item.url, filterOptions: item.filterOptions || {} });
+
+      expect(data).toMatchSnapshot();
+    });
+  });
+});

package/packages/web-crawler/src/utils/htmlToMarkdown.ts

@@ -0,0 +1,45 @@
+import { Readability } from '@mozilla/readability';
+import { Window } from 'happy-dom';
+import { NodeHtmlMarkdown, type TranslatorConfigObject } from 'node-html-markdown';
+
+import { FilterOptions } from '../type';
+
+export const htmlToMarkdown = (
+  html: string,
+  { url, filterOptions }: { filterOptions: FilterOptions; url: string },
+) => {
+  const window = new Window({ url });
+
+  const document = window.document;
+  document.body.innerHTML = html;
+
+  // @ts-expect-error reason: Readability expects a Document type
+  const parsedContent = new Readability(document).parse();
+
+  const useReadability = filterOptions.enableReadability ?? true;
+
+  let htmlNode = html;
+
+  if (useReadability && parsedContent?.content) {
+    htmlNode = parsedContent?.content;
+  }
+
+  const customTranslators = (
+    filterOptions.pureText
+      ? {
+          a: {
+            postprocess: (_: string, content: string) => content,
+          },
+          img: {
+            ignore: true,
+          },
+        }
+      : {}
+  ) as TranslatorConfigObject;
+
+  const nodeHtmlMarkdown = new NodeHtmlMarkdown({}, customTranslators);
+
+  const content = nodeHtmlMarkdown.translate(htmlNode);
+
+  return { ...parsedContent, content };
+};

package/packages/web-crawler/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "module": "CommonJS",
+    "target": "ESNext",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "sourceMap": true,
+    "skipDefaultLibCheck": true,
+    "jsx": "react-jsx",
+    "baseUrl": ".",
+    "allowSyntheticDefaultImports": true,
+    "moduleResolution": "node",
+    "forceConsistentCasingInFileNames": true,
+    "noImplicitReturns": true,
+    "noUnusedLocals": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "types": ["vitest/globals"]
+  }
+}

package/pnpm-workspace.yaml

@@ -0,0 +1,3 @@
+packages:
+  - 'packages/**'
+  - '.'

package/src/features/Conversation/Messages/Assistant/Tool/Render/CustomRender.tsx

@@ -1,14 +1,9 @@
-import { Icon } from '@lobehub/ui';
-import { ConfigProvider, Empty } from 'antd';
-import { useTheme } from 'antd-style';
-import { LucideSquareArrowLeft, LucideSquareArrowRight } from 'lucide-react';
-import { memo, useContext, useEffect } from 'react';
-import { useTranslation } from 'react-i18next';
-import { Center, Flexbox } from 'react-layout-kit';
+import { memo, useEffect } from 'react';
+import { Flexbox } from 'react-layout-kit';
 
 import PluginRender from '@/features/PluginsUI/Render';
 import { useChatStore } from '@/store/chat';
-import { chatPortalSelectors, chatSelectors } from '@/store/chat/selectors';
+import { chatSelectors } from '@/store/chat/selectors';
 import { ChatMessage } from '@/types/message';
 
 import Arguments from './Arguments';
@@ -30,14 +25,7 @@ const CustomRender = memo<CustomRenderProps>(
     setShowPluginRender,
     pluginError,
   }) => {
-    const [loading, isMessageToolUIOpen] = useChatStore((s) => [
-      chatSelectors.isPluginApiInvoking(id)(s),
-      chatPortalSelectors.isPluginUIOpen(id)(s),
-    ]);
-    const { direction } = useContext(ConfigProvider.ConfigContext);
-    const { t } = useTranslation('plugin');
-
-    const theme = useTheme();
+    const [loading] = useChatStore((s) => [chatSelectors.isPluginApiInvoking(id)(s)]);
 
     useEffect(() => {
       if (!plugin?.type || loading) return;
@@ -45,25 +33,6 @@ const CustomRender = memo<CustomRenderProps>(
       setShowPluginRender(plugin?.type !== 'default');
     }, [plugin?.type, loading]);
 
-    if (isMessageToolUIOpen)
-      return (
-        <Center paddingBlock={8} style={{ background: theme.colorFillQuaternary, borderRadius: 4 }}>
-          <Empty
-            description={t('showInPortal')}
-            image={
-              <Icon
-                color={theme.colorTextQuaternary}
-                icon={direction === 'rtl' ? LucideSquareArrowLeft : LucideSquareArrowRight}
-                size={'large'}
-              />
-            }
-            styles={{
-              image: { height: 24 },
-            }}
-          />
-        </Center>
-      );
-
     if (loading) return <Arguments arguments={requestArgs} shine />;
 
     return (

package/src/features/Conversation/Messages/Assistant/Tool/Render/index.tsx

@@ -1,12 +1,12 @@
 import { Suspense, memo } from 'react';
 
 import { LOADING_FLAT } from '@/const/message';
-import ErrorResponse from '@/features/Conversation/Messages/Assistant/Tool/Render/ErrorResponse';
 import { useChatStore } from '@/store/chat';
 import { chatSelectors } from '@/store/chat/selectors';
 
 import Arguments from './Arguments';
 import CustomRender from './CustomRender';
+import ErrorResponse from './ErrorResponse';
 
 interface RenderProps {
   messageId: string;

package/src/features/PluginsUI/Render/BuiltinType/index.tsx

@@ -7,6 +7,7 @@ import Loading from '../Loading';
 import { useParseContent } from '../useParseContent';
 
 export interface BuiltinTypeProps {
+  apiName?: string;
   arguments?: string;
   content: string;
   id: string;
@@ -25,6 +26,7 @@ const BuiltinType = memo<BuiltinTypeProps>(
     identifier,
     loading,
     pluginError,
+    apiName,
   }) => {
     const { isJSON, data } = useParseContent(content);
 
@@ -40,6 +42,7 @@ const BuiltinType = memo<BuiltinTypeProps>(
 
     return (
       <Render
+        apiName={apiName}
         args={args}
         content={data}
         identifier={identifier}

package/src/features/PluginsUI/Render/index.tsx

@@ -40,6 +40,7 @@ const PluginRender = memo<PluginRenderProps>(
       case 'builtin': {
         return (
           <BuiltinType
+            apiName={payload?.apiName}
             arguments={argumentsStr}
             content={content}
             id={id}

package/src/features/Portal/Plugins/Body/ToolRender.tsx

@@ -40,6 +40,7 @@ const ToolRender = memo(() => {
 
   return (
     <Render
+      apiName={plugin.apiName}
       arguments={args}
       identifier={plugin.identifier}
       messageId={messageId}

package/src/locales/default/plugin.ts

@@ -141,6 +141,18 @@ export default {
       close: '删除',
       confirm: '已完成配置并重试',
     },
+    crawPages: {
+      crawling: '链接识别中',
+      detail: {
+        preview: '预览',
+        raw: '原始文本',
+        tooLong: '文本内容过长，对话上下文仅保留前 10000 字符，超过部分不计入会话上下文',
+      },
+      meta: {
+        crawler: '抓取模式',
+        words: '字符数',
+      },
+    },
     searchxng: {
       baseURL: '请输入',
       description: '请输入 SearchXNG 的网址，即可开始联网搜索',

package/src/server/routers/tools/search.ts

@@ -1,4 +1,6 @@
+import { Crawler } from '@lobechat/web-crawler';
 import { TRPCError } from '@trpc/server';
+import pMap from 'p-map';
 import { z } from 'zod';
 
 import { toolsEnv } from '@/config/tools';
@@ -10,6 +12,27 @@ import { SEARCH_SEARXNG_NOT_CONFIG } from '@/types/tool/search';
 const searchProcedure = isServerMode ? authedProcedure : passwordProcedure;
 
 export const searchRouter = router({
+  crawlPages: searchProcedure
+    .input(
+      z.object({
+        impls: z.string().array().optional(),
+        urls: z.string().array(),
+      }),
+    )
+    .mutation(async ({ input }) => {
+      const crawler = new Crawler();
+
+      const results = await pMap(
+        input.urls,
+        async (url) => {
+          return await crawler.crawl({ impls: input.impls, url });
+        },
+        { concurrency: 10 },
+      );
+
+      return { results };
+    }),
+
   query: searchProcedure
     .input(
       z.object({

package/src/services/search.ts

@@ -4,6 +4,14 @@ class SearchService {
   search(query: string, searchEngine?: string[]) {
     return toolsClient.search.query.query({ query, searchEngine });
   }
+
+  crawlPage(url: string) {
+    return toolsClient.search.crawlPages.mutate({ urls: [url] });
+  }
+
+  crawlPages(urls: string[]) {
+    return toolsClient.search.crawlPages.mutate({ urls });
+  }
 }
 
 export const searchService = new SearchService();

package/src/store/chat/slices/builtinTool/actions/searXNG.ts

@@ -13,6 +13,16 @@ import {
 import { nanoid } from '@/utils/uuid';
 
 export interface SearchAction {
+  crawlMultiPages: (
+    id: string,
+    params: { urls: string[] },
+    aiSummary?: boolean,
+  ) => Promise<boolean | undefined>;
+  crawlSinglePage: (
+    id: string,
+    params: { url: string },
+    aiSummary?: boolean,
+  ) => Promise<boolean | undefined>;
   /**
    * 重新发起搜索
    * @description 会更新插件的 arguments 参数，然后再次搜索
@@ -28,6 +38,7 @@ export interface SearchAction {
     data: SearchQuery,
     aiSummary?: boolean,
   ) => Promise<void | boolean>;
+  togglePageContent: (url: string) => void;
   toggleSearchLoading: (id: string, loading: boolean) => void;
 }
 
@@ -37,12 +48,47 @@ export const searchSlice: StateCreator<
   [],
   SearchAction
 > = (set, get) => ({
+  crawlMultiPages: async (id, params, aiSummary = true) => {
+    const { internal_updateMessageContent } = get();
+    get().toggleSearchLoading(id, true);
+    const response = await searchService.crawlPages(params.urls);
+
+    await get().updatePluginState(id, response);
+    get().toggleSearchLoading(id, false);
+    const { results } = response;
+
+    if (!results) return;
+
+    const content = results.map((item) =>
+      'errorMessage' in item
+        ? item
+        : {
+            ...item.data,
+            // if crawl too many content
+            // slice the top 10000 char
+            content: item.data.content?.slice(0, 10_000),
+          },
+    );
+
+    await internal_updateMessageContent(id, JSON.stringify(content));
+
+    // if aiSummary is true, then trigger ai message
+    return aiSummary;
+  },
+
+  crawlSinglePage: async (id, params, aiSummary) => {
+    const { crawlMultiPages } = get();
+
+    return await crawlMultiPages(id, { urls: [params.url] }, aiSummary);
+  },
+
   reSearchWithSearXNG: async (id, data, options) => {
     get().toggleSearchLoading(id, true);
     await get().updatePluginArguments(id, data);
 
     await get().searchWithSearXNG(id, data, options?.aiSummary);
   },
+
   saveSearXNGSearchResult: async (id) => {
     const message = chatSelectors.getMessageById(id)(get());
     if (!message || !message.plugin) return;
@@ -133,6 +179,10 @@ export const searchSlice: StateCreator<
     return aiSummary;
   },
 
+  togglePageContent: (url) => {
+    set({ activePageContentUrl: url });
+  },
+
   toggleSearchLoading: (id, loading) => {
     set(
       { searchLoading: { ...get().searchLoading, [id]: loading } },

package/src/store/chat/slices/builtinTool/initialState.ts

@@ -1,6 +1,7 @@
 import { FileItem } from '@/types/files';
 
 export interface ChatToolState {
+  activePageContentUrl?: string;
   dalleImageLoading: Record<string, boolean>;
   dalleImageMap: Record<string, FileItem>;
   searchLoading: Record<string, boolean>;

package/src/tools/web-browsing/Portal/PageContent/index.tsx

@@ -0,0 +1,190 @@
+import { Alert, CopyButton, Icon, Markdown } from '@lobehub/ui';
+import { Descriptions, Segmented, Typography } from 'antd';
+import { createStyles } from 'antd-style';
+import { ExternalLink } from 'lucide-react';
+import Link from 'next/link';
+import { memo, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Flexbox } from 'react-layout-kit';
+
+import { CrawlResult } from '@/types/tool/crawler';
+
+const useStyles = createStyles(({ token, css }) => {
+  return {
+    cardBody: css`
+      padding-block: 12px 8px;
+      padding-inline: 16px;
+    `,
+    container: css`
+      cursor: pointer;
+
+      overflow: hidden;
+
+      max-width: 360px;
+      border: 1px solid ${token.colorBorderSecondary};
+      border-radius: 12px;
+
+      transition: border-color 0.2s;
+
+      :hover {
+        border-color: ${token.colorPrimary};
+      }
+    `,
+    description: css`
+      margin-block: 0 4px !important;
+      color: ${token.colorTextSecondary};
+    `,
+    detailsSection: css`
+      padding-block: ${token.paddingSM}px;
+    `,
+    externalLink: css`
+      color: ${token.colorPrimary};
+    `,
+    footer: css`
+      padding: ${token.paddingXS}px;
+      border-radius: 6px;
+      text-align: center;
+      background-color: ${token.colorFillQuaternary};
+    `,
+    footerText: css`
+      font-size: ${token.fontSizeSM}px;
+      color: ${token.colorTextTertiary} !important;
+    `,
+    metaInfo: css`
+      display: flex;
+      align-items: center;
+      color: ${token.colorTextSecondary};
+    `,
+    sliced: css`
+      color: ${token.colorTextQuaternary};
+    `,
+    title: css`
+      overflow: hidden;
+      display: -webkit-box;
+      -webkit-box-orient: vertical;
+      -webkit-line-clamp: 2;
+
+      margin-block-end: 0;
+
+      font-size: 16px;
+      font-weight: bold;
+    `,
+    titleRow: css`
+      color: ${token.colorText};
+    `,
+
+    url: css`
+      color: ${token.colorTextTertiary};
+    `,
+  };
+});
+
+interface PageContentProps {
+  messageId: string;
+  result?: CrawlResult;
+}
+
+const SLICED_LIMITED = 10_000;
+
+const PageContent = memo<PageContentProps>(({ result }) => {
+  const { t } = useTranslation('plugin');
+  const { styles } = useStyles();
+  const [display, setDisplay] = useState('render');
+
+  if (!result) return undefined;
+
+  const { url, title, description, content } = result.data;
+  return (
+    <Flexbox gap={24}>
+      <Flexbox gap={8}>
+        <Flexbox
+          align={'center'}
+          className={styles.titleRow}
+          gap={24}
+          horizontal
+          justify={'space-between'}
+        >
+          <Flexbox>
+            <div className={styles.title}>{title || result.originalUrl}</div>
+          </Flexbox>
+        </Flexbox>
+        {description && (
+          <Typography.Paragraph
+            className={styles.description}
+            ellipsis={{ expandable: false, rows: 4 }}
+          >
+            {description}
+          </Typography.Paragraph>
+        )}
+        <Flexbox align={'center'} className={styles.url} gap={4} horizontal>
+          {result.data.siteName && <div>{result.data.siteName} · </div>}
+          <Link
+            className={styles.url}
+            href={url}
+            onClick={(e) => e.stopPropagation()}
+            rel={'nofollow'}
+            style={{ display: 'flex', gap: 4 }}
+            target={'_blank'}
+          >
+            {result.originalUrl}
+            <Icon icon={ExternalLink} />
+          </Link>
+        </Flexbox>
+
+        <div className={styles.footer}>
+          <Descriptions
+            classNames={{
+              content: styles.footerText,
+            }}
+            column={2}
+            items={[
+              {
+                children: result.data.content?.length,
+                label: t('search.crawPages.meta.words'),
+              },
+              {
+                children: result.crawler,
+                label: t('search.crawPages.meta.crawler'),
+              },
+            ]}
+            size="small"
+          />
+        </div>
+      </Flexbox>
+      {content && (
+        <Flexbox gap={12} paddingBlock={'0 12px'}>
+          <Flexbox horizontal justify={'space-between'}>
+            <Segmented
+              onChange={(value) => setDisplay(value)}
+              options={[
+                { label: t('search.crawPages.detail.preview'), value: 'render' },
+                { label: t('search.crawPages.detail.raw'), value: 'raw' },
+              ]}
+              value={display}
+            />
+            <CopyButton content={content} />
+          </Flexbox>
+          {content.length > SLICED_LIMITED && (
+            <Alert message={t('search.crawPages.detail.tooLong')} variant={'pure'} />
+          )}
+          {display === 'render' ? (
+            <Markdown variant={'chat'}>{content}</Markdown>
+          ) : (
+            <div style={{ paddingBlock: '0 12px' }}>
+              {content.length < SLICED_LIMITED ? (
+                content
+              ) : (
+                <>
+                  <span>{content.slice(0, SLICED_LIMITED)}</span>
+                  <span className={styles.sliced}>{content.slice(SLICED_LIMITED, -1)}</span>
+                </>
+              )}
+            </div>
+          )}
+        </Flexbox>
+      )}
+    </Flexbox>
+  );
+});
+
+export default PageContent;

package/src/tools/web-browsing/Portal/PageContents/index.tsx

@@ -0,0 +1,23 @@
+import { memo } from 'react';
+
+import { useChatStore } from '@/store/chat';
+import { CrawlResult } from '@/types/tool/crawler';
+
+import PageContent from '../PageContent';
+
+interface PageContentProps {
+  messageId: string;
+  results: CrawlResult[];
+  urls: string[];
+}
+
+const PageContents = memo<PageContentProps>(({ urls, messageId, results }) => {
+  const activePageContentUrl = useChatStore((s) => s.activePageContentUrl);
+
+  const url = urls.find((u) => u === activePageContentUrl);
+  const result = results.find((result) => result.originalUrl === url);
+
+  return <PageContent messageId={messageId} result={result} />;
+});
+
+export default PageContents;

package/src/tools/web-browsing/Portal/{ResultList → Search/ResultList}/SearchItem/Video.tsx

@@ -5,7 +5,7 @@ import { Flexbox } from 'react-layout-kit';
 
 import { SearchResult } from '@/types/tool/search';
 
-import { ENGINE_ICON_MAP } from '../../../const';
+import { ENGINE_ICON_MAP } from '../../../../const';
 import TitleExtra from './TitleExtra';
 
 const useStyles = createStyles(({ css, token }) => {

package/src/tools/web-browsing/Portal/Search/index.tsx

@@ -0,0 +1,69 @@
+import { Skeleton } from 'antd';
+import { uniq } from 'lodash-es';
+import { memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { useChatStore } from '@/store/chat';
+import { chatToolSelectors } from '@/store/chat/selectors';
+import { SearchQuery, SearchResponse } from '@/types/tool/search';
+
+import SearchBar from '../../components/SearchBar';
+import Footer from './Footer';
+import ResultList from './ResultList';
+
+interface InspectorUIProps {
+  messageId: string;
+  query: SearchQuery;
+  response: SearchResponse;
+}
+
+const Inspector = memo<InspectorUIProps>(({ query: args, messageId, response }) => {
+  const engines = uniq((response.results || []).map((result) => result.engine));
+  const defaultEngines = engines.length > 0 ? engines : args.searchEngines || [];
+  const loading = useChatStore(chatToolSelectors.isSearXNGSearching(messageId));
+
+  if (loading) {
+    return (
+      <Flexbox gap={12} height={'100%'}>
+        <SearchBar
+          aiSummary={false}
+          defaultEngines={defaultEngines}
+          defaultQuery={args.query}
+          messageId={messageId}
+          tooltip={false}
+        />
+
+        <Flexbox gap={16} paddingBlock={16} paddingInline={12}>
+          {[1, 2, 3, 4, 6].map((id) => (
+            <Skeleton
+              active
+              key={id}
+              paragraph={{ rows: 3, width: `${(id % 4) + 5}0%` }}
+              title={false}
+            />
+          ))}
+        </Flexbox>
+      </Flexbox>
+    );
+  }
+
+  return (
+    <Flexbox gap={0} height={'100%'}>
+      <Flexbox gap={12} height={'100%'}>
+        <SearchBar
+          aiSummary={false}
+          defaultEngines={defaultEngines}
+          defaultQuery={args.query}
+          messageId={messageId}
+          tooltip={false}
+        />
+        <Flexbox height={'100%'} width={'100%'}>
+          <ResultList dataSources={response.results} />
+        </Flexbox>
+      </Flexbox>
+      <Footer />
+    </Flexbox>
+  );
+});
+
+export default Inspector;