@lobehub/chat 0.27.1 → 0.27.3

package/CHANGELOG.md

@@ -2,6 +2,64 @@
 
 # Changelog
 
+### [Version 0.27.3](https://github.com/lobehub/lobe-chat/compare/v0.27.2...v0.27.3)
+
+<sup>Released on **2023-07-29**</sup>
+
+#### 🐛 Bug Fixes
+
+- **misc**: 修正返回结果导致插件无法正常识别的问题.
+
+#### 💄 Styles
+
+- **misc**: 优化样式.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's fixed
+
+- **misc**: 修正返回结果导致插件无法正常识别的问题 ([b183188](https://github.com/lobehub/lobe-chat/commit/b183188))
+
+#### Styles
+
+- **misc**: 优化样式 ([9ce5d1d](https://github.com/lobehub/lobe-chat/commit/9ce5d1d))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
+### [Version 0.27.2](https://github.com/lobehub/lobe-chat/compare/v0.27.1...v0.27.2)
+
+<sup>Released on **2023-07-29**</sup>
+
+#### ♻ Code Refactoring
+
+- **misc**: 重构并优化文档抓取插件能力.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### Code refactoring
+
+- **misc**: 重构并优化文档抓取插件能力 ([ff56348](https://github.com/lobehub/lobe-chat/commit/ff56348))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 0.27.1](https://github.com/lobehub/lobe-chat/compare/v0.27.0...v0.27.1)
 
 <sup>Released on **2023-07-29**</sup>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "0.27.1",
+  "version": "0.27.3",
   "description": "Lobe Chat is an open-source chatbot client using LangChain, Typescript and Next.js",
   "keywords": [
     "chatbot",

package/src/pages/chat/[id]/Conversation/ChatList/Plugins/FunctionCall.tsx

@@ -49,7 +49,6 @@ const FunctionCall = memo<FunctionCallProps>(({ function_call, loading }) => {
           avatar
         )}
         {t(`plugins.${function_call?.name}` as any, { ns: 'plugin' })}
-        {loading ? `（${t('loading.plugin')}）` : null}
         <Icon icon={open ? LucideChevronUp : LucideChevronDown} />
       </Flexbox>
       {open && <Highlighter language={'json'}>{args}</Highlighter>}

package/src/plugins/webCrawler/runner.ts

@@ -1,50 +1,45 @@
 import { PluginRunner } from '@/plugins/type';
 
-import { DataResults, Result } from './type';
+import { ParserResponse, Result } from './type';
 
-const BASE_URL =
-  'https://api.apify.com/v2/acts/apify~website-content-crawler/run-sync-get-dataset-items';
-const token = process.env.APIFY_API_KEY;
+const BASE_URL = process.env.BROWSERLESS_URL ?? 'https://chrome.browserless.io';
+const BROWSERLESS_TOKEN = process.env.BROWSERLESS_TOKEN;
+
+// service from: https://github.com/lobehub/html-parser/tree/master
+const HTML_PARSER_URL = process.env.HTML_PARSER_URL;
 
 const runner: PluginRunner<{ url: string }, Result> = async ({ url }) => {
-  // Prepare Actor input
   const input = {
-    aggressivePrune: false,
-    clickElementsCssSelector: '[aria-expanded="false"]',
-    debugMode: false,
-    dynamicContentWaitSecs: 3,
-    proxyConfiguration: {
-      useApifyProxy: true,
-    },
-    removeCookieWarnings: true,
-    removeElementsCssSelector:
-      'nav, footer, script, style, noscript, svg,\n[role="alert"],\n[role="banner"],\n[role="dialog"],\n[role="alertdialog"],\n[role="region"][aria-label*="skip" i],\n[aria-modal="true"]',
-    saveFiles: false,
-    saveHtml: false,
-    saveMarkdown: true,
-    saveScreenshots: false,
-    startUrls: [{ url }],
+    gotoOptions: { waitUntil: 'networkidle2' },
+    url,
   };
 
   try {
-    const data = await fetch(`${BASE_URL}?token=${token}`, {
+    const res = await fetch(`${BASE_URL}/content?token=${BROWSERLESS_TOKEN}`, {
       body: JSON.stringify(input),
       headers: {
         'Content-Type': 'application/json',
       },
       method: 'POST',
     });
-    const result = (await data.json()) as DataResults;
-
-    const item = result[0];
-    return {
-      content: item.markdown,
-      title: item.metadata.title,
-      url: item.url,
-    };
+    const html = await res.text();
+
+    const parserBody = { html, url };
+
+    const parseRes = await fetch(`${HTML_PARSER_URL}`, {
+      body: JSON.stringify(parserBody),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      method: 'POST',
+    });
+
+    const { title, textContent, siteName } = (await parseRes.json()) as ParserResponse;
+
+    return { content: textContent, title, url, website: siteName };
   } catch (error) {
     console.error(error);
-    return { content: '抓取失败', errorMessage: (error as any).message };
+    return { content: '抓取失败', errorMessage: (error as any).message, url };
   }
 };
 

package/src/plugins/webCrawler/type.ts

@@ -1,32 +1,35 @@
-export type DataResults = DataItem[];
-
 export type Result = {
-  content?: string;
+  content: string;
   title?: string;
-  url?: string;
-};
-export interface DataItem {
-  crawl: Crawl;
-  markdown: string;
-  metadata: Metadata;
-  screenshotUrl: any;
-  text: string;
   url: string;
-}
+  website?: string;
+};
 
-export interface Crawl {
-  depth: number;
-  httpStatusCode: number;
-  loadedTime: string;
-  loadedUrl: string;
-  referrerUrl: string;
-}
+export interface ParserResponse {
+  /** author metadata */
+  byline: string;
+
+  /** HTML string of processed article content */
+  content: string;
+
+  /** content direction */
+  dir: string;
+
+  /** article description, or short excerpt from the content */
+  excerpt: string;
+
+  /** content language */
+  lang: string;
+
+  /** length of an article, in characters */
+  length: number;
+
+  /** name of the site */
+  siteName: string;
+
+  /** text content of the article, with all the HTML tags removed */
+  textContent: string;
 
-export interface Metadata {
-  author: any;
-  canonicalUrl: string;
-  description: string;
-  keywords: string;
-  languageCode: string;
+  /** article title */
   title: string;
 }