@lobehub/chat 1.66.6 → 1.67.0

package/src/tools/web-browsing/Portal/index.tsx

@@ -1,77 +1,41 @@
-import { Skeleton } from 'antd';
-import { uniq } from 'lodash-es';
 import { memo } from 'react';
-import { Flexbox } from 'react-layout-kit';
 
-import { useChatStore } from '@/store/chat';
-import { chatToolSelectors } from '@/store/chat/selectors';
-import { SearchResponse } from '@/types/tool/search';
+import { WebBrowsingApiName } from '@/tools/web-browsing';
+import { BuiltinPortalProps } from '@/types/tool';
+import { CrawlPluginState } from '@/types/tool/crawler';
+import { SearchQuery } from '@/types/tool/search';
 
-import SearchBar from '../components/SearchBar';
-import Footer from './Footer';
-import ResultList from './ResultList';
+import PageContent from './PageContent';
+import PageContents from './PageContents';
+import Search from './Search';
 
-interface SearchArguments {
-  query: string;
-  searchEngine?: string[];
-}
+const Inspector = memo<BuiltinPortalProps>(({ arguments: args, messageId, state, apiName }) => {
+  switch (apiName) {
+    case WebBrowsingApiName.searchWithSearXNG: {
+      return <Search messageId={messageId} query={args as SearchQuery} response={state} />;
+    }
 
-interface InspectorUIProps<T = Record<string, any>, S = any> {
-  arguments: T;
-  identifier: string;
-  messageId: string;
-  state: S;
-}
+    case WebBrowsingApiName.crawlSinglePage: {
+      const url = args.url;
+      const result = (state as CrawlPluginState).results.find(
+        (result) => result.originalUrl === url,
+      );
 
-const Inspector = memo<InspectorUIProps<SearchArguments, SearchResponse>>(
-  ({ arguments: args, messageId, state }) => {
-    const engines = uniq((state.results || []).map((result) => result.engine));
-    const defaultEngines = engines.length > 0 ? engines : args.searchEngine || [];
-    const loading = useChatStore(chatToolSelectors.isSearXNGSearching(messageId));
+      return <PageContent messageId={messageId} result={result} />;
+    }
 
-    if (loading) {
+    case WebBrowsingApiName.crawlMultiPages: {
       return (
-        <Flexbox gap={12} height={'100%'}>
-          <SearchBar
-            aiSummary={false}
-            defaultEngines={defaultEngines}
-            defaultQuery={args.query}
-            messageId={messageId}
-            tooltip={false}
-          />
-
-          <Flexbox gap={16} paddingBlock={16} paddingInline={12}>
-            {[1, 2, 3, 4, 6].map((id) => (
-              <Skeleton
-                active
-                key={id}
-                paragraph={{ rows: 3, width: `${(id % 4) + 5}0%` }}
-                title={false}
-              />
-            ))}
-          </Flexbox>
-        </Flexbox>
+        <PageContents
+          messageId={messageId}
+          results={(state as CrawlPluginState).results}
+          urls={args.urls}
+        />
       );
     }
+  }
 
-    return (
-      <Flexbox gap={0} height={'100%'}>
-        <Flexbox gap={12} height={'100%'}>
-          <SearchBar
-            aiSummary={false}
-            defaultEngines={defaultEngines}
-            defaultQuery={args.query}
-            messageId={messageId}
-            tooltip={false}
-          />
-          <Flexbox height={'100%'} width={'100%'}>
-            <ResultList dataSources={state.results} />
-          </Flexbox>
-        </Flexbox>
-        <Footer />
-      </Flexbox>
-    );
-  },
-);
+  return null;
+});
 
 export default Inspector;

package/src/tools/web-browsing/Render/PageContent/Loading.tsx

@@ -0,0 +1,57 @@
+'use client';
+
+import { CopyButton } from '@lobehub/ui';
+import { createStyles } from 'antd-style';
+import Link from 'next/link';
+import { memo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Flexbox } from 'react-layout-kit';
+
+import { shinyTextStylish } from '@/styles/loading';
+
+const useStyles = createStyles(({ token, css }) => {
+  return {
+    cardBody: css`
+      padding-block: 12px 8px;
+      padding-inline: 12px;
+    `,
+    container: css`
+      overflow: hidden;
+      justify-content: space-between;
+
+      max-width: 360px;
+      border: 1px solid ${token.colorBorderSecondary};
+      border-radius: 12px;
+    `,
+
+    footer: css`
+      padding-block: 8px;
+      padding-inline: 12px;
+
+      font-size: ${token.fontSizeSM}px;
+      color: ${token.colorTextTertiary};
+
+      background-color: ${token.colorFillQuaternary};
+    `,
+    shining: shinyTextStylish(token),
+  };
+});
+
+const LoadingCard = memo<{ url: string }>(({ url }) => {
+  const { t } = useTranslation('plugin');
+  const { styles } = useStyles();
+
+  return (
+    <Flexbox className={styles.container}>
+      <Flexbox className={styles.cardBody} horizontal>
+        <Link href={url} rel={'nofollow'} target={'_blank'}>
+          <div className={styles.shining}>{url}</div>
+        </Link>
+        <CopyButton content={url} size={'small'} />
+      </Flexbox>
+      <div className={styles.footer}>{t('search.crawPages.crawling')}</div>
+    </Flexbox>
+  );
+});
+
+export default LoadingCard;

package/src/tools/web-browsing/Render/PageContent/Result.tsx

@@ -0,0 +1,142 @@
+'use client';
+
+import { CrawlSuccessResult } from '@lobechat/web-crawler';
+import { Icon } from '@lobehub/ui';
+import { Descriptions, Typography } from 'antd';
+import { createStyles } from 'antd-style';
+import { ExternalLink } from 'lucide-react';
+import Link from 'next/link';
+import { memo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Center, Flexbox } from 'react-layout-kit';
+
+import { useChatStore } from '@/store/chat';
+import { WebBrowsingManifest } from '@/tools/web-browsing';
+
+const { Paragraph } = Typography;
+
+const useStyles = createStyles(({ token, css }) => {
+  return {
+    cardBody: css`
+      padding-block: 12px 8px;
+      padding-inline: 16px;
+    `,
+    container: css`
+      cursor: pointer;
+
+      overflow: hidden;
+
+      max-width: 360px;
+      border: 1px solid ${token.colorBorderSecondary};
+      border-radius: 12px;
+
+      transition: border-color 0.2s;
+
+      :hover {
+        border-color: ${token.colorPrimary};
+      }
+    `,
+    description: css`
+      margin-block: 0 4px !important;
+      color: ${token.colorTextTertiary};
+    `,
+    detailsSection: css`
+      padding-block: ${token.paddingSM}px;
+    `,
+    externalLink: css`
+      color: ${token.colorTextQuaternary};
+
+      :hover {
+        color: ${token.colorText};
+      }
+    `,
+    footer: css`
+      padding: ${token.paddingXS}px;
+      text-align: center;
+      background-color: ${token.colorFillQuaternary};
+    `,
+    footerText: css`
+      font-size: ${token.fontSizeSM}px;
+      color: ${token.colorTextTertiary} !important;
+    `,
+    metaInfo: css`
+      display: flex;
+      align-items: center;
+      color: ${token.colorTextSecondary};
+    `,
+    title: css`
+      overflow: hidden;
+      display: -webkit-box;
+      -webkit-box-orient: vertical;
+      -webkit-line-clamp: 1;
+
+      margin-block-end: 0;
+    `,
+    titleRow: css`
+      color: ${token.colorText};
+    `,
+  };
+});
+
+interface CrawlerData {
+  crawler: string;
+  messageId: string;
+  originalUrl: string;
+  result: CrawlSuccessResult;
+}
+
+const CrawlerResultCard = memo<CrawlerData>(({ result, messageId, crawler, originalUrl }) => {
+  const { t } = useTranslation('plugin');
+  const { styles } = useStyles();
+  const [openToolUI, togglePageContent] = useChatStore((s) => [s.openToolUI, s.togglePageContent]);
+
+  const { url, title, description } = result;
+
+  return (
+    <Flexbox
+      className={styles.container}
+      justify={'space-between'}
+      onClick={() => {
+        openToolUI(messageId, WebBrowsingManifest.identifier);
+        togglePageContent(originalUrl);
+      }}
+    >
+      <Flexbox className={styles.cardBody} gap={8}>
+        <Flexbox align={'center'} className={styles.titleRow} horizontal justify={'space-between'}>
+          <Flexbox>
+            <div className={styles.title}>{title || originalUrl}</div>
+          </Flexbox>
+          <Link href={url} onClick={(e) => e.stopPropagation()} target={'_blank'}>
+            <Center className={styles.externalLink}>
+              <Icon icon={ExternalLink} />
+            </Center>
+          </Link>
+        </Flexbox>
+        <Paragraph className={styles.description} ellipsis={{ expandable: false, rows: 2 }}>
+          {description || result.content?.slice(0, 40)}
+        </Paragraph>
+      </Flexbox>
+      <div className={styles.footer}>
+        <Descriptions
+          classNames={{
+            content: styles.footerText,
+          }}
+          column={2}
+          items={[
+            {
+              children: result.content?.length,
+              label: t('search.crawPages.meta.words'),
+            },
+            {
+              children: crawler,
+              label: t('search.crawPages.meta.crawler'),
+            },
+          ]}
+          size="small"
+        />
+      </div>
+    </Flexbox>
+  );
+});
+
+export default CrawlerResultCard;

package/src/tools/web-browsing/Render/PageContent/index.tsx

@@ -0,0 +1,41 @@
+import { memo } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { CrawlPluginState } from '@/types/tool/crawler';
+
+import Loading from './Loading';
+import Result from './Result';
+
+interface PagesContentProps {
+  messageId: string;
+  results?: CrawlPluginState['results'];
+  urls: string[];
+}
+
+const PagesContent = memo<PagesContentProps>(({ results, messageId, urls }) => {
+  if (!results || results.length === 0) {
+    return (
+      <Flexbox gap={12} horizontal>
+        {urls.map((url) => (
+          <Loading key={url} url={url} />
+        ))}
+      </Flexbox>
+    );
+  }
+
+  return (
+    <Flexbox gap={12} horizontal>
+      {results.map((result) => (
+        <Result
+          crawler={result.crawler}
+          key={result.originalUrl}
+          messageId={messageId}
+          originalUrl={result.originalUrl}
+          result={result.data}
+        />
+      ))}
+    </Flexbox>
+  );
+});
+
+export default PagesContent;

package/src/tools/web-browsing/Render/{SearchQuery → Search/SearchQuery}/SearchView.tsx

@@ -8,7 +8,7 @@ import { Flexbox } from 'react-layout-kit';
 
 import { useIsMobile } from '@/hooks/useIsMobile';
 
-import { EngineAvatarGroup } from '../../components/EngineAvatar';
+import { EngineAvatarGroup } from '../../../components/EngineAvatar';
 
 const useStyles = createStyles(({ css, token }) => ({
   font: css`

package/src/tools/web-browsing/Render/{SearchQuery → Search/SearchQuery}/index.tsx

@@ -10,7 +10,7 @@ import { useChatStore } from '@/store/chat';
 import { chatToolSelectors } from '@/store/chat/selectors';
 import { SearchQuery, SearchResponse } from '@/types/tool/search';
 
-import SearchBar from '../../components/SearchBar';
+import SearchBar from '../../../components/SearchBar';
 import SearchView from './SearchView';
 
 interface SearchQueryViewProps {

package/src/tools/web-browsing/Render/{SearchResult → Search/SearchResult}/ShowMore.tsx

@@ -6,7 +6,7 @@ import { Flexbox } from 'react-layout-kit';
 import { useChatStore } from '@/store/chat';
 import { WebBrowsingManifest } from '@/tools/web-browsing';
 
-import { EngineAvatarGroup } from '../../components/EngineAvatar';
+import { EngineAvatarGroup } from '../../../components/EngineAvatar';
 
 const useStyles = createStyles(({ css, token }) => ({
   container: css`

package/src/tools/web-browsing/Render/Search/index.tsx

@@ -0,0 +1,62 @@
+import { Alert, Highlighter } from '@lobehub/ui';
+import { memo, useState } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { ChatMessagePluginError } from '@/types/message';
+import { SearchQuery, SearchResponse } from '@/types/tool/search';
+
+import ConfigForm from './ConfigForm';
+import SearchQueryView from './SearchQuery';
+import SearchResult from './SearchResult';
+
+interface SearchProps {
+  messageId: string;
+  pluginError: ChatMessagePluginError;
+  searchQuery: SearchQuery;
+  searchResponse?: SearchResponse;
+}
+
+const Search = memo<SearchProps>(({ messageId, searchQuery, searchResponse, pluginError }) => {
+  const [editing, setEditing] = useState(false);
+
+  if (pluginError) {
+    if (pluginError?.type === 'PluginSettingsInvalid') {
+      return <ConfigForm id={messageId} provider={pluginError.body?.provider} />;
+    }
+
+    return (
+      <Alert
+        extra={
+          <Flexbox>
+            <Highlighter copyButtonSize={'small'} language={'json'} type={'pure'}>
+              {JSON.stringify(pluginError.body?.data || pluginError.body, null, 2)}
+            </Highlighter>
+          </Flexbox>
+        }
+        message={pluginError?.message}
+        type={'error'}
+      />
+    );
+  }
+
+  return (
+    <Flexbox gap={8}>
+      <SearchQueryView
+        args={searchQuery}
+        editing={editing}
+        messageId={messageId}
+        pluginState={searchResponse}
+        setEditing={setEditing}
+      />
+      <SearchResult
+        args={searchQuery}
+        editing={editing}
+        messageId={messageId}
+        pluginState={searchResponse}
+        setEditing={setEditing}
+      />
+    </Flexbox>
+  );
+});
+
+export default Search;

package/src/tools/web-browsing/Render/index.tsx

@@ -1,56 +1,47 @@
-import { Alert, Highlighter } from '@lobehub/ui';
-import { memo, useState } from 'react';
-import { Flexbox } from 'react-layout-kit';
+import { memo } from 'react';
 
+import { WebBrowsingApiName } from '@/tools/web-browsing';
+import PageContent from '@/tools/web-browsing/Render/PageContent';
 import { BuiltinRenderProps } from '@/types/tool';
+import { CrawlMultiPagesQuery, CrawlPluginState, CrawlSinglePageQuery } from '@/types/tool/crawler';
 import { SearchContent, SearchQuery, SearchResponse } from '@/types/tool/search';
 
-import ConfigForm from './ConfigForm';
-import SearchQueryView from './SearchQuery';
-import SearchResult from './SearchResult';
+import Search from './Search';
 
-const WebBrowsing = memo<BuiltinRenderProps<SearchContent[], SearchQuery, SearchResponse>>(
-  ({ messageId, args, pluginState, pluginError }) => {
-    const [editing, setEditing] = useState(false);
+const WebBrowsing = memo<BuiltinRenderProps<SearchContent[]>>(
+  ({ messageId, args, pluginState, pluginError, apiName }) => {
+    switch (apiName) {
+      case WebBrowsingApiName.searchWithSearXNG: {
+        return (
+          <Search
+            messageId={messageId}
+            pluginError={pluginError}
+            searchQuery={args as SearchQuery}
+            searchResponse={pluginState as SearchResponse}
+          />
+        );
+      }
 
-    if (pluginError) {
-      if (pluginError?.type === 'PluginSettingsInvalid') {
-        return <ConfigForm id={messageId} provider={pluginError.body?.provider} />;
+      case WebBrowsingApiName.crawlSinglePage: {
+        return (
+          <PageContent
+            messageId={messageId}
+            results={(pluginState as CrawlPluginState)?.results}
+            urls={[(args as CrawlSinglePageQuery).url]}
+          />
+        );
       }
 
-      return (
-        <Alert
-          extra={
-            <Flexbox>
-              <Highlighter copyButtonSize={'small'} language={'json'} type={'pure'}>
-                {JSON.stringify(pluginError.body?.data || pluginError.body, null, 2)}
-              </Highlighter>
-            </Flexbox>
-          }
-          message={pluginError?.message}
-          type={'error'}
-        />
-      );
+      case WebBrowsingApiName.crawlMultiPages: {
+        return (
+          <PageContent
+            messageId={messageId}
+            results={(pluginState as CrawlPluginState)?.results}
+            urls={(args as CrawlMultiPagesQuery).urls}
+          />
+        );
+      }
     }
-
-    return (
-      <Flexbox gap={8}>
-        <SearchQueryView
-          args={args}
-          editing={editing}
-          messageId={messageId}
-          pluginState={pluginState}
-          setEditing={setEditing}
-        />
-        <SearchResult
-          args={args}
-          editing={editing}
-          messageId={messageId}
-          pluginState={pluginState}
-          setEditing={setEditing}
-        />
-      </Flexbox>
-    );
   },
 );
 

package/src/tools/web-browsing/index.ts

@@ -2,12 +2,20 @@ import dayjs from 'dayjs';
 
 import { BuiltinToolManifest } from '@/types/tool';
 
+import { systemPrompt } from './systemRole';
+
+export const WebBrowsingApiName = {
+  crawlMultiPages: 'crawlMultiPages',
+  crawlSinglePage: 'crawlSinglePage',
+  searchWithSearXNG: 'searchWithSearXNG',
+};
+
 export const WebBrowsingManifest: BuiltinToolManifest = {
   api: [
     {
       description:
         'A meta search engine. Useful for when you need to answer questions about current events. Input should be a search query. Output is a JSON array of the query results',
-      name: 'searchWithSearXNG',
+      name: WebBrowsingApiName.searchWithSearXNG,
       parameters: {
         properties: {
           query: {
@@ -46,57 +54,45 @@ export const WebBrowsingManifest: BuiltinToolManifest = {
         type: 'object',
       },
     },
+    {
+      description:
+        'A crawler can visit page content. Output is a JSON object of title, content, url and website',
+      name: WebBrowsingApiName.crawlSinglePage,
+      parameters: {
+        properties: {
+          url: {
+            description: 'The url need to be crawled',
+            type: 'string',
+          },
+        },
+        required: ['url'],
+        type: 'object',
+      },
+    },
+    {
+      description:
+        'A crawler can visit multi pages. If need to visit multi website, use this one. Output is an array of JSON object of title, content, url and website',
+      name: WebBrowsingApiName.crawlMultiPages,
+      parameters: {
+        properties: {
+          urls: {
+            items: {
+              description: 'The url need to be crawled',
+              type: 'string',
+            },
+            type: 'array',
+          },
+        },
+        required: ['urls'],
+        type: 'object',
+      },
+    },
   ],
   identifier: 'lobe-web-browsing',
   meta: {
     avatar: '🌐',
     title: 'Web Browsing',
   },
-  systemRole: `You are a search tool that uses SearxNG, a meta search engine. When given a search query, you will return relevant results from various search engines. Your results will be in JSON format, containing titles, links, and snippets of relevant web pages. If no good results are found, you will say so. You can also provide answers, infoboxes, or suggestions if available.
-
-SearXNG combine with these engines:
-- Google: The world's most popular search engine, offering a wide range of general web results.,
-- Bilibili: A Chinese video sharing website themed around animation, comic, and games (ACG) ，（又名 B 站）,
-- Bing: Microsoft's search engine, providing web results with a focus on visual search.,
-- DuckDuckGo: A privacy-focused search engine that doesn't track users.,
-- npm: The package manager for JavaScript, used to find Node.js packages.,
-- PyPI: The Python Package Index, used to find Python packages.,
-- GitHub: A platform for version control and collaboration, search for code repositories.,
-- arXiv: A repository of electronic preprints for scientific papers.,
-- Google Scholar: A freely accessible web search engine for scholarly literature.,
-- Z-Library: A shadow library project for file-sharing access to scholarly journal articles and books.,
-- Reddit: A network of communities based on people's interests, search for discussions and content.,
-- IMDb: An online database of information related to films, TV programs, and video games.,
-- Brave: A privacy-focused browser with its own search engine.,
-- Wikipedia: A free online encyclopedia, search for articles on various topics.,
-- Pinterest: An image sharing and social media service, search for images and ideas.,
-- Unsplash: A website dedicated to sharing stock photography, search for high-quality images.,
-- Vimeo: A video hosting, sharing, and services platform.,
-- YouTube: A video sharing platform, search for a wide variety of video content.
-
-SearXNG comes with a search syntax by with you can modify the categories, engines, languages and more.
-
-## \`!\` select engine and category
-
-To set category and/or engine names use a \`!\` prefix. To give a few examples:
-
-- search in category **map** for **paris**
-  - \`!map paris\`
-- image search
-  - \`!images Wau Holland\`
-
-Abbreviations of the engines and languages are also accepted. Engine/category modifiers are chain able and inclusive. E.g. with \`!map !ddg !wp paris\` search in map category and DuckDuckGo and Wikipedia for **paris**.
-
-## \`:\` select language
-
-To select language filter use a \`:\` prefix. To give an example:
-
-- search Wikipedia by a custom language
-  - \`:fr !wp Wau Holland\`
-
-You need to summarize in the language of the user's question. If you use search content in your reply, you must use Markdown footnote format to indicate the source, Such as [^1].
-
-current date: ${dayjs(new Date()).format('YYYY-MM-DD')}
-`,
+  systemRole: systemPrompt(dayjs(new Date()).format('YYYY-MM-DD')),
   type: 'builtin',
 };