@lobehub/chat 0.13.1 → 0.14.0

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+## [Version 0.14.0](https://github.com/lobehub/lobe-chat/compare/v0.13.1...v0.14.0)
+
+<sup>Released on **2023-07-24**</sup>
+
+#### ✨ Features
+
+- **misc**: 支持网页抓取.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's improved
+
+- **misc**: 支持网页抓取, closes [#14](https://github.com/lobehub/lobe-chat/issues/14) ([9e933b0](https://github.com/lobehub/lobe-chat/commit/9e933b0))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 0.13.1](https://github.com/lobehub/lobe-chat/compare/v0.13.0...v0.13.1)
 
 <sup>Released on **2023-07-23**</sup>

package/README.md

@@ -66,12 +66,12 @@ $ pnpm start
 
 > 📊 Total: <kbd>**4**</kbd>
 
-<a href="https://github.com/canisminor1990" title="canisminor1990">
-  <img src="https://avatars.githubusercontent.com/u/17870709?v=4" width="50" />
-</a>
 <a href="https://github.com/arvinxx" title="arvinxx">
   <img src="https://avatars.githubusercontent.com/u/28616219?v=4" width="50" />
 </a>
+<a href="https://github.com/canisminor1990" title="canisminor1990">
+  <img src="https://avatars.githubusercontent.com/u/17870709?v=4" width="50" />
+</a>
 <a href="https://github.com/apps/dependabot" title="dependabot[bot]">
   <img src="https://avatars.githubusercontent.com/in/29110?v=4" width="50" />
 </a>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "0.13.1",
+  "version": "0.14.0",
   "description": "Lobe Chat is an open-source chatbot client using LangChain, Typescript and Next.js",
   "keywords": [
     "chatbot",

package/src/locales/default/common.ts

@@ -37,6 +37,8 @@ export default {
   'ok': '确定',
   'plugin-realtimeWeather': '实时天气预报',
   'plugin-searchEngine': '搜索引擎',
+  'plugin-undefined': '插件检测中...',
+  'plugin-websiteCrawler': '网页内容提取',
   'pluginList': '插件列表',
   'pluginLoading': '插件运行中...',
   'profile': '助手身份',

package/src/pages/chat/[id]/Conversation/MessageExtra.tsx

@@ -69,7 +69,7 @@ const MessageExtra = ({ role, extra, function_call }: ChatMessage): ReactNode =>
       if (!(hasModelTag || hasFuncTag)) return;
 
       return (
-        <Flexbox className={styles.container}>
+        <Flexbox className={styles.container} horizontal>
           {hasFuncTag && funcTag}
           {hasModelTag && modelTag}
         </Flexbox>

package/src/plugins/index.ts

@@ -1,6 +1,7 @@
 import searchEngine from './searchEngine';
 import getWeather from './weather';
+import webCrawler from './webCrawler';
 
-const pluginList = [getWeather, searchEngine];
+const pluginList = [getWeather, searchEngine, webCrawler];
 
 export default pluginList;

package/src/plugins/webCrawler/index.ts

@@ -0,0 +1,20 @@
+import runner from './runner';
+
+const schema = {
+  description: '提取网页内容并总结',
+  name: 'websiteCrawler',
+  parameters: {
+    properties: {
+      url: {
+        description: '网页内容',
+        type: 'string',
+      },
+    },
+    required: ['url'],
+    type: 'object',
+  },
+};
+
+const getWeather = { avatar: '🕸', name: 'websiteCrawler', runner, schema };
+
+export default getWeather;

package/src/plugins/webCrawler/runner.ts

@@ -0,0 +1,75 @@
+export type DataResults = DataItem[];
+
+export interface DataItem {
+  crawl: Crawl;
+  markdown: string;
+  metadata: Metadata;
+  screenshotUrl: any;
+  text: string;
+  url: string;
+}
+
+export interface Crawl {
+  depth: number;
+  httpStatusCode: number;
+  loadedTime: string;
+  loadedUrl: string;
+  referrerUrl: string;
+}
+
+export interface Metadata {
+  author: any;
+  canonicalUrl: string;
+  description: string;
+  keywords: string;
+  languageCode: string;
+  title: string;
+}
+
+const BASE_URL =
+  'https://api.apify.com/v2/acts/apify~website-content-crawler/run-sync-get-dataset-items';
+const token = process.env.APIFY_API_KEY;
+
+const runner = async ({ url }: { url: string }) => {
+  // Prepare Actor input
+  const input = {
+    aggressivePrune: false,
+    clickElementsCssSelector: '[aria-expanded="false"]',
+    debugMode: false,
+    dynamicContentWaitSecs: 3,
+    proxyConfiguration: {
+      useApifyProxy: true,
+    },
+    removeCookieWarnings: true,
+    removeElementsCssSelector:
+      'nav, footer, script, style, noscript, svg,\n[role="alert"],\n[role="banner"],\n[role="dialog"],\n[role="alertdialog"],\n[role="region"][aria-label*="skip" i],\n[aria-modal="true"]',
+    saveFiles: false,
+    saveHtml: false,
+    saveMarkdown: true,
+    saveScreenshots: false,
+    startUrls: [{ url }],
+  };
+
+  try {
+    const data = await fetch(`${BASE_URL}?token=${token}`, {
+      body: JSON.stringify(input),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      method: 'POST',
+    });
+    const result = (await data.json()) as DataResults;
+
+    const item = result[0];
+    return {
+      content: item.markdown,
+      title: item.metadata.title,
+      url: item.url,
+    };
+  } catch (error) {
+    console.error(error);
+    return { content: '抓取失败', errorMessage: (error as any).message };
+  }
+};
+
+export default runner;