@lobehub/chat 0.27.1 → 0.27.2

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+### [Version 0.27.2](https://github.com/lobehub/lobe-chat/compare/v0.27.1...v0.27.2)
+
+<sup>Released on **2023-07-29**</sup>
+
+#### ♻ Code Refactoring
+
+- **misc**: 重构并优化文档抓取插件能力.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### Code refactoring
+
+- **misc**: 重构并优化文档抓取插件能力 ([ff56348](https://github.com/lobehub/lobe-chat/commit/ff56348))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 0.27.1](https://github.com/lobehub/lobe-chat/compare/v0.27.0...v0.27.1)
 
 <sup>Released on **2023-07-29**</sup>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "0.27.1",
+  "version": "0.27.2",
   "description": "Lobe Chat is an open-source chatbot client using LangChain, Typescript and Next.js",
   "keywords": [
     "chatbot",

package/src/plugins/webCrawler/runner.ts

@@ -1,47 +1,40 @@
 import { PluginRunner } from '@/plugins/type';
 
-import { DataResults, Result } from './type';
+import { Result } from './type';
 
-const BASE_URL =
-  'https://api.apify.com/v2/acts/apify~website-content-crawler/run-sync-get-dataset-items';
-const token = process.env.APIFY_API_KEY;
+const BASE_URL = process.env.BROWSERLESS_URL ?? 'https://chrome.browserless.io';
+const BROWSERLESS_TOKEN = process.env.BROWSERLESS_TOKEN;
+
+// service from: https://github.com/lobehub/html-parser/tree/master
+const HTML_PARSER_URL = process.env.HTML_PARSER_URL;
 
 const runner: PluginRunner<{ url: string }, Result> = async ({ url }) => {
-  // Prepare Actor input
   const input = {
-    aggressivePrune: false,
-    clickElementsCssSelector: '[aria-expanded="false"]',
-    debugMode: false,
-    dynamicContentWaitSecs: 3,
-    proxyConfiguration: {
-      useApifyProxy: true,
-    },
-    removeCookieWarnings: true,
-    removeElementsCssSelector:
-      'nav, footer, script, style, noscript, svg,\n[role="alert"],\n[role="banner"],\n[role="dialog"],\n[role="alertdialog"],\n[role="region"][aria-label*="skip" i],\n[aria-modal="true"]',
-    saveFiles: false,
-    saveHtml: false,
-    saveMarkdown: true,
-    saveScreenshots: false,
-    startUrls: [{ url }],
+    gotoOptions: { waitUntil: 'networkidle2' },
+    url,
   };
 
   try {
-    const data = await fetch(`${BASE_URL}?token=${token}`, {
+    const res = await fetch(`${BASE_URL}/content?token=${BROWSERLESS_TOKEN}`, {
       body: JSON.stringify(input),
       headers: {
         'Content-Type': 'application/json',
       },
       method: 'POST',
     });
-    const result = (await data.json()) as DataResults;
-
-    const item = result[0];
-    return {
-      content: item.markdown,
-      title: item.metadata.title,
-      url: item.url,
-    };
+    const html = await res.text();
+
+    const parserBody = { html, url };
+
+    const parseRes = await fetch(`${HTML_PARSER_URL}`, {
+      body: JSON.stringify(parserBody),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      method: 'POST',
+    });
+
+    return await parseRes.json();
   } catch (error) {
     console.error(error);
     return { content: '抓取失败', errorMessage: (error as any).message };

package/src/plugins/webCrawler/type.ts

@@ -1,32 +1,5 @@
-export type DataResults = DataItem[];
-
 export type Result = {
   content?: string;
   title?: string;
   url?: string;
 };
-export interface DataItem {
-  crawl: Crawl;
-  markdown: string;
-  metadata: Metadata;
-  screenshotUrl: any;
-  text: string;
-  url: string;
-}
-
-export interface Crawl {
-  depth: number;
-  httpStatusCode: number;
-  loadedTime: string;
-  loadedUrl: string;
-  referrerUrl: string;
-}
-
-export interface Metadata {
-  author: any;
-  canonicalUrl: string;
-  description: string;
-  keywords: string;
-  languageCode: string;
-  title: string;
-}