@lobehub/chat 0.27.0 → 0.27.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/package.json +1 -1
  3. package/src/plugins/searchEngine/Render.tsx +33 -32
  4. package/src/plugins/webCrawler/runner.ts +22 -29
  5. package/src/plugins/webCrawler/type.ts +0 -27
package/CHANGELOG.md CHANGED
@@ -2,6 +2,56 @@
2
2
 
3
3
  # Changelog
4
4
 
5
+ ### [Version 0.27.2](https://github.com/lobehub/lobe-chat/compare/v0.27.1...v0.27.2)
6
+
7
+ <sup>Released on **2023-07-29**</sup>
8
+
9
+ #### ♻ Code Refactoring
10
+
11
+ - **misc**: 重构并优化文档抓取插件能力.
12
+
13
+ <br/>
14
+
15
+ <details>
16
+ <summary><kbd>Improvements and Fixes</kbd></summary>
17
+
18
+ #### Code refactoring
19
+
20
+ - **misc**: 重构并优化文档抓取插件能力 ([ff56348](https://github.com/lobehub/lobe-chat/commit/ff56348))
21
+
22
+ </details>
23
+
24
+ <div align="right">
25
+
26
+ [![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
27
+
28
+ </div>
29
+
30
+ ### [Version 0.27.1](https://github.com/lobehub/lobe-chat/compare/v0.27.0...v0.27.1)
31
+
32
+ <sup>Released on **2023-07-29**</sup>
33
+
34
+ #### 💄 Styles
35
+
36
+ - **misc**: 优化搜索引擎样式.
37
+
38
+ <br/>
39
+
40
+ <details>
41
+ <summary><kbd>Improvements and Fixes</kbd></summary>
42
+
43
+ #### Styles
44
+
45
+ - **misc**: 优化搜索引擎样式 ([699afb3](https://github.com/lobehub/lobe-chat/commit/699afb3))
46
+
47
+ </details>
48
+
49
+ <div align="right">
50
+
51
+ [![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
52
+
53
+ </div>
54
+
5
55
  ## [Version 0.27.0](https://github.com/lobehub/lobe-chat/compare/v0.26.1...v0.27.0)
6
56
 
7
57
  <sup>Released on **2023-07-29**</sup>
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lobehub/chat",
3
- "version": "0.27.0",
3
+ "version": "0.27.2",
4
4
  "description": "Lobe Chat is an open-source chatbot client using LangChain, Typescript and Next.js",
5
5
  "keywords": [
6
6
  "chatbot",
package/src/plugins/searchEngine/Render.tsx CHANGED
@@ -41,44 +41,45 @@ const useStyles = createStyles(({ css, token, cx }) => {
41
41
  const Render = memo<PluginRenderProps<Result>>(({ content }) => {
42
42
  const { styles } = useStyles();
43
43
  return (
44
- <Flexbox gap={8}>
45
- <Flexbox>
46
- {content.map((item) => (
47
- <Flexbox
48
- className={styles.container}
49
- distribution={'space-between'}
50
- horizontal
51
- key={item.link}
52
- padding={12}
53
- >
54
- <Flexbox>
55
- <Link className={styles.link} href={item.link!} target={'_blank'}>
56
- <Flexbox>
57
- <Flexbox align={'center'} gap={12} horizontal>
58
- {item.favicon && (
59
- <img
60
- alt={item.title || item.link}
61
- className={styles.favicon}
62
- height={24}
63
- src={item.favicon}
64
- width={24}
65
- />
66
- )}
67
- <Flexbox>
68
- <Flexbox className={styles.desc}>{item.source}</Flexbox>
69
- <Flexbox className={styles.displayLink}>{item.displayed_link}</Flexbox>
70
- </Flexbox>
44
+ <Flexbox>
45
+ {content.map((item, index) => (
46
+ <Flexbox
47
+ className={styles.container}
48
+ distribution={'space-between'}
49
+ horizontal
50
+ key={item.link}
51
+ padding={12}
52
+ >
53
+ <Flexbox>
54
+ <Link className={styles.link} href={item.link!} target={'_blank'}>
55
+ <Flexbox>
56
+ <Flexbox align={'center'} gap={12} horizontal>
57
+ {item.favicon && (
58
+ <img
59
+ alt={item.title || item.link}
60
+ className={styles.favicon}
61
+ height={24}
62
+ src={item.favicon}
63
+ width={24}
64
+ />
65
+ )}
66
+ <Flexbox>
67
+ <Flexbox className={styles.desc}>{item.source}</Flexbox>
68
+ <Flexbox className={styles.displayLink}>{item.displayed_link}</Flexbox>
71
69
  </Flexbox>
70
+ </Flexbox>
71
+ <Flexbox align={'baseline'} gap={2} horizontal>
72
+ <div>{index + 1}.</div>
72
73
  <Flexbox className={styles.title}>{item.title}</Flexbox>
73
74
  </Flexbox>
74
- </Link>
75
- <Flexbox className={styles.desc}>
76
- {item.date ? `${item.date} - ${item.content}` : item.content}
77
75
  </Flexbox>
76
+ </Link>
77
+ <Flexbox className={styles.desc}>
78
+ {item.date ? `${item.date} - ${item.content}` : item.content}
78
79
  </Flexbox>
79
80
  </Flexbox>
80
- ))}
81
- </Flexbox>
81
+ </Flexbox>
82
+ ))}
82
83
  </Flexbox>
83
84
  );
84
85
  });
package/src/plugins/webCrawler/runner.ts CHANGED
@@ -1,47 +1,40 @@
1
1
  import { PluginRunner } from '@/plugins/type';
2
2
 
3
- import { DataResults, Result } from './type';
3
+ import { Result } from './type';
4
4
 
5
- const BASE_URL =
6
- 'https://api.apify.com/v2/acts/apify~website-content-crawler/run-sync-get-dataset-items';
7
- const token = process.env.APIFY_API_KEY;
5
+ const BASE_URL = process.env.BROWSERLESS_URL ?? 'https://chrome.browserless.io';
6
+ const BROWSERLESS_TOKEN = process.env.BROWSERLESS_TOKEN;
7
+
8
+ // service from: https://github.com/lobehub/html-parser/tree/master
9
+ const HTML_PARSER_URL = process.env.HTML_PARSER_URL;
8
10
 
9
11
  const runner: PluginRunner<{ url: string }, Result> = async ({ url }) => {
10
- // Prepare Actor input
11
12
  const input = {
12
- aggressivePrune: false,
13
- clickElementsCssSelector: '[aria-expanded="false"]',
14
- debugMode: false,
15
- dynamicContentWaitSecs: 3,
16
- proxyConfiguration: {
17
- useApifyProxy: true,
18
- },
19
- removeCookieWarnings: true,
20
- removeElementsCssSelector:
21
- 'nav, footer, script, style, noscript, svg,\n[role="alert"],\n[role="banner"],\n[role="dialog"],\n[role="alertdialog"],\n[role="region"][aria-label*="skip" i],\n[aria-modal="true"]',
22
- saveFiles: false,
23
- saveHtml: false,
24
- saveMarkdown: true,
25
- saveScreenshots: false,
26
- startUrls: [{ url }],
13
+ gotoOptions: { waitUntil: 'networkidle2' },
14
+ url,
27
15
  };
28
16
 
29
17
  try {
30
- const data = await fetch(`${BASE_URL}?token=${token}`, {
18
+ const res = await fetch(`${BASE_URL}/content?token=${BROWSERLESS_TOKEN}`, {
31
19
  body: JSON.stringify(input),
32
20
  headers: {
33
21
  'Content-Type': 'application/json',
34
22
  },
35
23
  method: 'POST',
36
24
  });
37
- const result = (await data.json()) as DataResults;
38
-
39
- const item = result[0];
40
- return {
41
- content: item.markdown,
42
- title: item.metadata.title,
43
- url: item.url,
44
- };
25
+ const html = await res.text();
26
+
27
+ const parserBody = { html, url };
28
+
29
+ const parseRes = await fetch(`${HTML_PARSER_URL}`, {
30
+ body: JSON.stringify(parserBody),
31
+ headers: {
32
+ 'Content-Type': 'application/json',
33
+ },
34
+ method: 'POST',
35
+ });
36
+
37
+ return await parseRes.json();
45
38
  } catch (error) {
46
39
  console.error(error);
47
40
  return { content: '抓取失败', errorMessage: (error as any).message };
package/src/plugins/webCrawler/type.ts CHANGED
@@ -1,32 +1,5 @@
1
- export type DataResults = DataItem[];
2
-
3
1
  export type Result = {
4
2
  content?: string;
5
3
  title?: string;
6
4
  url?: string;
7
5
  };
8
- export interface DataItem {
9
- crawl: Crawl;
10
- markdown: string;
11
- metadata: Metadata;
12
- screenshotUrl: any;
13
- text: string;
14
- url: string;
15
- }
16
-
17
- export interface Crawl {
18
- depth: number;
19
- httpStatusCode: number;
20
- loadedTime: string;
21
- loadedUrl: string;
22
- referrerUrl: string;
23
- }
24
-
25
- export interface Metadata {
26
- author: any;
27
- canonicalUrl: string;
28
- description: string;
29
- keywords: string;
30
- languageCode: string;
31
- title: string;
32
- }