@lobehub/chat 1.66.5 → 1.67.0

package/src/tools/web-browsing/Render/Search/index.tsx

@@ -0,0 +1,62 @@
+import { Alert, Highlighter } from '@lobehub/ui';
+import { memo, useState } from 'react';
+import { Flexbox } from 'react-layout-kit';
+
+import { ChatMessagePluginError } from '@/types/message';
+import { SearchQuery, SearchResponse } from '@/types/tool/search';
+
+import ConfigForm from './ConfigForm';
+import SearchQueryView from './SearchQuery';
+import SearchResult from './SearchResult';
+
+interface SearchProps {
+  messageId: string;
+  pluginError: ChatMessagePluginError;
+  searchQuery: SearchQuery;
+  searchResponse?: SearchResponse;
+}
+
+const Search = memo<SearchProps>(({ messageId, searchQuery, searchResponse, pluginError }) => {
+  const [editing, setEditing] = useState(false);
+
+  if (pluginError) {
+    if (pluginError?.type === 'PluginSettingsInvalid') {
+      return <ConfigForm id={messageId} provider={pluginError.body?.provider} />;
+    }
+
+    return (
+      <Alert
+        extra={
+          <Flexbox>
+            <Highlighter copyButtonSize={'small'} language={'json'} type={'pure'}>
+              {JSON.stringify(pluginError.body?.data || pluginError.body, null, 2)}
+            </Highlighter>
+          </Flexbox>
+        }
+        message={pluginError?.message}
+        type={'error'}
+      />
+    );
+  }
+
+  return (
+    <Flexbox gap={8}>
+      <SearchQueryView
+        args={searchQuery}
+        editing={editing}
+        messageId={messageId}
+        pluginState={searchResponse}
+        setEditing={setEditing}
+      />
+      <SearchResult
+        args={searchQuery}
+        editing={editing}
+        messageId={messageId}
+        pluginState={searchResponse}
+        setEditing={setEditing}
+      />
+    </Flexbox>
+  );
+});
+
+export default Search;

package/src/tools/web-browsing/Render/index.tsx

@@ -1,56 +1,47 @@
-import { Alert, Highlighter } from '@lobehub/ui';
-import { memo, useState } from 'react';
-import { Flexbox } from 'react-layout-kit';
+import { memo } from 'react';
 
+import { WebBrowsingApiName } from '@/tools/web-browsing';
+import PageContent from '@/tools/web-browsing/Render/PageContent';
 import { BuiltinRenderProps } from '@/types/tool';
+import { CrawlMultiPagesQuery, CrawlPluginState, CrawlSinglePageQuery } from '@/types/tool/crawler';
 import { SearchContent, SearchQuery, SearchResponse } from '@/types/tool/search';
 
-import ConfigForm from './ConfigForm';
-import SearchQueryView from './SearchQuery';
-import SearchResult from './SearchResult';
+import Search from './Search';
 
-const WebBrowsing = memo<BuiltinRenderProps<SearchContent[], SearchQuery, SearchResponse>>(
-  ({ messageId, args, pluginState, pluginError }) => {
-    const [editing, setEditing] = useState(false);
+const WebBrowsing = memo<BuiltinRenderProps<SearchContent[]>>(
+  ({ messageId, args, pluginState, pluginError, apiName }) => {
+    switch (apiName) {
+      case WebBrowsingApiName.searchWithSearXNG: {
+        return (
+          <Search
+            messageId={messageId}
+            pluginError={pluginError}
+            searchQuery={args as SearchQuery}
+            searchResponse={pluginState as SearchResponse}
+          />
+        );
+      }
 
-    if (pluginError) {
-      if (pluginError?.type === 'PluginSettingsInvalid') {
-        return <ConfigForm id={messageId} provider={pluginError.body?.provider} />;
+      case WebBrowsingApiName.crawlSinglePage: {
+        return (
+          <PageContent
+            messageId={messageId}
+            results={(pluginState as CrawlPluginState)?.results}
+            urls={[(args as CrawlSinglePageQuery).url]}
+          />
+        );
       }
 
-      return (
-        <Alert
-          extra={
-            <Flexbox>
-              <Highlighter copyButtonSize={'small'} language={'json'} type={'pure'}>
-                {JSON.stringify(pluginError.body?.data || pluginError.body, null, 2)}
-              </Highlighter>
-            </Flexbox>
-          }
-          message={pluginError?.message}
-          type={'error'}
-        />
-      );
+      case WebBrowsingApiName.crawlMultiPages: {
+        return (
+          <PageContent
+            messageId={messageId}
+            results={(pluginState as CrawlPluginState)?.results}
+            urls={(args as CrawlMultiPagesQuery).urls}
+          />
+        );
+      }
     }
-
-    return (
-      <Flexbox gap={8}>
-        <SearchQueryView
-          args={args}
-          editing={editing}
-          messageId={messageId}
-          pluginState={pluginState}
-          setEditing={setEditing}
-        />
-        <SearchResult
-          args={args}
-          editing={editing}
-          messageId={messageId}
-          pluginState={pluginState}
-          setEditing={setEditing}
-        />
-      </Flexbox>
-    );
   },
 );
 

package/src/tools/web-browsing/index.ts

@@ -2,12 +2,20 @@ import dayjs from 'dayjs';
 
 import { BuiltinToolManifest } from '@/types/tool';
 
+import { systemPrompt } from './systemRole';
+
+export const WebBrowsingApiName = {
+  crawlMultiPages: 'crawlMultiPages',
+  crawlSinglePage: 'crawlSinglePage',
+  searchWithSearXNG: 'searchWithSearXNG',
+};
+
 export const WebBrowsingManifest: BuiltinToolManifest = {
   api: [
     {
       description:
         'A meta search engine. Useful for when you need to answer questions about current events. Input should be a search query. Output is a JSON array of the query results',
-      name: 'searchWithSearXNG',
+      name: WebBrowsingApiName.searchWithSearXNG,
       parameters: {
         properties: {
           query: {
@@ -46,57 +54,45 @@ export const WebBrowsingManifest: BuiltinToolManifest = {
         type: 'object',
       },
     },
+    {
+      description:
+        'A crawler can visit page content. Output is a JSON object of title, content, url and website',
+      name: WebBrowsingApiName.crawlSinglePage,
+      parameters: {
+        properties: {
+          url: {
+            description: 'The url need to be crawled',
+            type: 'string',
+          },
+        },
+        required: ['url'],
+        type: 'object',
+      },
+    },
+    {
+      description:
+        'A crawler can visit multi pages. If need to visit multi website, use this one. Output is an array of JSON object of title, content, url and website',
+      name: WebBrowsingApiName.crawlMultiPages,
+      parameters: {
+        properties: {
+          urls: {
+            items: {
+              description: 'The url need to be crawled',
+              type: 'string',
+            },
+            type: 'array',
+          },
+        },
+        required: ['urls'],
+        type: 'object',
+      },
+    },
   ],
   identifier: 'lobe-web-browsing',
   meta: {
     avatar: '🌐',
     title: 'Web Browsing',
   },
-  systemRole: `You are a search tool that uses SearxNG, a meta search engine. When given a search query, you will return relevant results from various search engines. Your results will be in JSON format, containing titles, links, and snippets of relevant web pages. If no good results are found, you will say so. You can also provide answers, infoboxes, or suggestions if available.
-
-SearXNG combine with these engines:
-- Google: The world's most popular search engine, offering a wide range of general web results.,
-- Bilibili: A Chinese video sharing website themed around animation, comic, and games (ACG) ，（又名 B 站）,
-- Bing: Microsoft's search engine, providing web results with a focus on visual search.,
-- DuckDuckGo: A privacy-focused search engine that doesn't track users.,
-- npm: The package manager for JavaScript, used to find Node.js packages.,
-- PyPI: The Python Package Index, used to find Python packages.,
-- GitHub: A platform for version control and collaboration, search for code repositories.,
-- arXiv: A repository of electronic preprints for scientific papers.,
-- Google Scholar: A freely accessible web search engine for scholarly literature.,
-- Z-Library: A shadow library project for file-sharing access to scholarly journal articles and books.,
-- Reddit: A network of communities based on people's interests, search for discussions and content.,
-- IMDb: An online database of information related to films, TV programs, and video games.,
-- Brave: A privacy-focused browser with its own search engine.,
-- Wikipedia: A free online encyclopedia, search for articles on various topics.,
-- Pinterest: An image sharing and social media service, search for images and ideas.,
-- Unsplash: A website dedicated to sharing stock photography, search for high-quality images.,
-- Vimeo: A video hosting, sharing, and services platform.,
-- YouTube: A video sharing platform, search for a wide variety of video content.
-
-SearXNG comes with a search syntax by with you can modify the categories, engines, languages and more.
-
-## \`!\` select engine and category
-
-To set category and/or engine names use a \`!\` prefix. To give a few examples:
-
-- search in category **map** for **paris**
-  - \`!map paris\`
-- image search
-  - \`!images Wau Holland\`
-
-Abbreviations of the engines and languages are also accepted. Engine/category modifiers are chain able and inclusive. E.g. with \`!map !ddg !wp paris\` search in map category and DuckDuckGo and Wikipedia for **paris**.
-
-## \`:\` select language
-
-To select language filter use a \`:\` prefix. To give an example:
-
-- search Wikipedia by a custom language
-  - \`:fr !wp Wau Holland\`
-
-You need to summarize in the language of the user's question. If you use search content in your reply, you must use Markdown footnote format to indicate the source, Such as [^1].
-
-current date: ${dayjs(new Date()).format('YYYY-MM-DD')}
-`,
+  systemRole: systemPrompt(dayjs(new Date()).format('YYYY-MM-DD')),
   type: 'builtin',
 };

package/src/tools/web-browsing/systemRole.ts

@@ -0,0 +1,109 @@
+export const systemPrompt = (
+  date: string,
+) => `You have a Web Information tool with powerful internet access capabilities. You can search across multiple search engines and extract content from web pages to provide users with accurate, comprehensive, and up-to-date information.
+
+<core_capabilities>
+1. Search the web using multiple search engines (searchWithSearXNG)
+2. Retrieve content from a specific webpage (crawlSinglePage)
+3. Retrieve content from multiple webpages simultaneously (crawlMultiPages)
+</core_capabilities>
+
+<workflow>
+1. Analyze the nature of the user's query (factual information, research, current events, etc.)
+2. Select the appropriate tool and search strategy based on the query type
+3. Execute searches or crawl operations to gather relevant information
+4. Synthesize information with proper attribution of sources
+5. Present findings in a clear, organized manner with appropriate citations
+</workflow>
+
+<tool_selection_guidelines>
+- For general information queries: Use searchWithSearXNG with the most relevant search engines
+- For detailed understanding of specific single page content: Use 'crawlSinglePage' on the most authoritative or relevant page from search results. If you need to visit multiple pages, prefer to use 'crawlMultiPages'
+- For multi-perspective information or comparative analysis: Use 'crawlMultiPages' on several different relevant sources
+</tool_selection_guidelines>
+
+<search_engine_selection>
+Choose search engines based on the query type:
+- General knowledge: google, bing, duckduckgo, brave, wikipedia
+- Academic/scientific information: google scholar, arxiv, z-library
+- Code/technical queries: google, github, npm, pypi
+- Videos: youtube, vimeo, bilibili
+- Images: unsplash, pinterest
+- Entertainment: imdb, reddit
+- For region-specific information, prefer search engines popular in that region
+</search_engine_selection>
+
+<citation_requirements>
+- Always cite sources using markdown footnote format (e.g., [^1])
+- List all referenced URLs at the end of your response
+- Clearly distinguish between quoted information and your own analysis
+- Respond in the same language as the user's query
+</citation_requirements>
+
+<response_format>
+When providing information from web searches:
+1. Start with a direct answer to the user's question when possible
+2. Provide relevant details from sources
+3. Include proper citations using footnotes
+4. List all sources at the end of your response
+5. For time-sensitive information, note when the information was retrieved
+
+Example:
+
+According to recent studies, global temperatures have risen by 1.1°C since pre-industrial times[^1].
+
+[^1]: [Climate Report in 2023](https://example.org/climate-report-2023)
+</response_format>
+
+<searxng_description>
+SearXNG is a metasearch engine that can leverage multiple search engines including:
+- Google: World's most popular search engine providing broad web results
+- Bilibili: Chinese video sharing website focused on animation, comics, and games (aka B-site)
+- Bing: Microsoft's search engine providing web results with emphasis on visual search
+- DuckDuckGo: Privacy-focused search engine that doesn't track users
+- npm: JavaScript package manager for finding Node.js packages
+- PyPI: Python Package Index for finding Python packages
+- GitHub: Version control and collaboration platform for searching code repositories
+- arXiv: Repository of electronic preprints of scientific papers
+- Google Scholar: Free web search engine for scholarly literature
+- Z-Library: File-sharing project for journal articles and books
+- Reddit: Network of communities based on people's interests
+- IMDb: Online database related to films, TV programs, and video games
+- Brave: Privacy-focused browser with its own search engine
+- Wikipedia: Free online encyclopedia with articles on various topics
+- Pinterest: Image sharing and social media service for finding images
+- Unsplash: Website dedicated to sharing high-quality stock photography
+- Vimeo: Video hosting, sharing, and service platform
+- YouTube: Video sharing platform for searching various video content
+
+  <search_syntax>
+  SearXNG has special search syntax to modify the categories, engines, and language of searches:
+
+  1. Use \`!\` to select engines and categories:
+     - Search for "paris" in the "map" category: \`!map paris\`
+     - Search for images: \`!images Wau Holland\`
+     - Chain multiple modifiers: \`!map !ddg !wp paris\` (searches for "paris" in the map category, DuckDuckGo, and Wikipedia)
+
+  2. Use \`:\` to select language:
+     - Search Wikipedia in a specific language: \`:fr !wp Wau Holland\` (uses French)
+  </search_syntax>
+</searxng_description>
+
+<crawling_best_practices>
+- Only crawl pages that are publicly accessible
+- When crawling multiple pages, crawl all relevant sources
+- Prioritize authoritative sources over user-generated content when appropriate
+- For controversial topics, crawl sources representing different perspectives
+- Verify information across multiple sources when possible
+- Consider the recency of information, especially for time-sensitive topics
+</crawling_best_practices>
+
+<error_handling>
+- If search returns no results, try alternative search terms or engines
+- If a page cannot be crawled, explain the issue to the user and suggest alternatives
+- For ambiguous queries, ask for clarification before conducting extensive searches
+- If information seems outdated, note this to the user and suggest searching for more recent sources
+</error_handling>
+
+Current date: ${date}
+`;

package/src/types/tool/builtin.ts

@@ -29,6 +29,7 @@ export interface LobeBuiltinTool {
 }
 
 export interface BuiltinRenderProps<Content = any, Arguments = any, State = any> {
+  apiName?: string;
   args: Arguments;
   content: Content;
   identifier?: string;
@@ -40,6 +41,7 @@ export interface BuiltinRenderProps<Content = any, Arguments = any, State = any>
 export type BuiltinRender = <T = any>(props: BuiltinRenderProps<T>) => ReactNode;
 
 export interface BuiltinPortalProps<Arguments = Record<string, any>, State = any> {
+  apiName?: string;
   arguments: Arguments;
   identifier: string;
   messageId: string;

package/src/types/tool/crawler.ts

@@ -0,0 +1,19 @@
+import { CrawlSuccessResult } from '@lobechat/web-crawler';
+
+export interface CrawlSinglePageQuery {
+  url: string;
+}
+
+export interface CrawlMultiPagesQuery {
+  urls: string[];
+}
+
+export interface CrawlResult {
+  crawler: string;
+  data: CrawlSuccessResult;
+  originalUrl: string;
+}
+
+export interface CrawlPluginState {
+  results: CrawlResult[];
+}

package/src/types/tool/search.ts

@@ -2,6 +2,7 @@ export interface SearchQuery {
   query: string;
   searchEngines?: string[];
 }
+
 export const SEARCH_SEARXNG_NOT_CONFIG = 'SearXNG is not configured';
 
 export interface SearchResponse {