@devflow-tools/knowledge-engine 0.8.9 → 0.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -13
- package/dist/fetcher.d.ts +3 -0
- package/dist/fetcher.d.ts.map +1 -1
- package/dist/fetcher.js +53 -2
- package/dist/fetcher.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-engine.d.ts.map +1 -1
- package/dist/knowledge-engine.js +8 -2
- package/dist/knowledge-engine.js.map +1 -1
- package/package.json +6 -4
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @devflow-tools/knowledge-engine
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
DevFlow 知识引擎 — 从技术文档站同步内容，BM25 搜索，支持 SPA 页面和静态 HTML。
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -15,25 +15,57 @@ import { KnowledgeEngine } from '@devflow-tools/knowledge-engine';
|
|
|
15
15
|
|
|
16
16
|
const engine = new KnowledgeEngine({ projectRoot: '/path/to/project' });
|
|
17
17
|
|
|
18
|
-
//
|
|
19
|
-
|
|
18
|
+
// 注册知识源
|
|
19
|
+
await engine.registerSource({
|
|
20
|
+
name: 'react',
|
|
21
|
+
urls: ['https://react.dev/reference'],
|
|
22
|
+
pluginName: 'react',
|
|
23
|
+
version: '19.0.0',
|
|
24
|
+
renderJs: true, // SPA 页面需要 Playwright 渲染
|
|
25
|
+
});
|
|
20
26
|
|
|
21
|
-
//
|
|
22
|
-
await engine.syncDocs('react');
|
|
27
|
+
// 同步文档
|
|
28
|
+
const results = await engine.syncDocs('react');
|
|
29
|
+
|
|
30
|
+
// 搜索知识库
|
|
31
|
+
const entries = await engine.search('useEffect cleanup');
|
|
23
32
|
```
|
|
24
33
|
|
|
25
34
|
## Features
|
|
26
35
|
|
|
27
|
-
-
|
|
28
|
-
-
|
|
29
|
-
- **
|
|
36
|
+
- **智能抓取** — 自动检测页面内容量：静态内容充足则用 HTTP fetch，SPA 页面自动升级到 Playwright 浏览器渲染
|
|
37
|
+
- **`renderJs` 标记** — 在 `KnowledgeSource` 中设置 `renderJs: true` 直达 Playwright 模式，跳过自动检测
|
|
38
|
+
- **BM25 搜索** — 基于关键词的文档搜索，支持 scope 过滤（react/vue/nest 等）
|
|
39
|
+
- **HTML → Markdown** — 使用 turndown 转换，标题级 chunk 分块保留文档结构
|
|
40
|
+
- **多源同步** — 从官方文档、GitHub 等 URL 列表批量抓取和更新
|
|
41
|
+
|
|
42
|
+
## Fetcher API
|
|
43
|
+
|
|
44
|
+
| 函数 | 说明 |
|
|
45
|
+
|------|------|
|
|
46
|
+
| `fetchHtml(url)` | HTTP fetch + 重试（最多 2 次），支持 429 限流处理 |
|
|
47
|
+
| `fetchHtmlBrowser(url)` | Playwright Chromium 浏览器渲染（用于 SPA 页面） |
|
|
48
|
+
| `fetchHtmlSmart(url)` | 智能模式：先 fetch → 内容 < 500 字符 → 自动升级到浏览器 |
|
|
49
|
+
| `closeBrowser()` | 关闭 Playwright 浏览器实例，释放资源 |
|
|
50
|
+
|
|
51
|
+
## Engine API
|
|
52
|
+
|
|
53
|
+
- `new KnowledgeEngine({ projectRoot })` — 构造函数
|
|
54
|
+
- `.registerSource(source)` — 注册/合并知识源（已存在则合并 URL 列表）
|
|
55
|
+
- `.syncDocs(pluginName?)` — 同步指定（或全部）知识源的文档
|
|
56
|
+
- `.search(query, options?)` — BM25 搜索知识库，支持 `scope` 过滤
|
|
57
|
+
- `.lookup(symbol, scope?)` — 精确匹配知识条目
|
|
58
|
+
- `.getSources()` — 列出所有知识源及其状态（chunk 数、最后同步时间）
|
|
30
59
|
|
|
31
|
-
##
|
|
60
|
+
## Environment
|
|
61
|
+
|
|
62
|
+
需要安装 Playwright Chromium：
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
npx playwright install chromium
|
|
66
|
+
```
|
|
32
67
|
|
|
33
|
-
|
|
34
|
-
- `.search(query)` — search knowledge base
|
|
35
|
-
- `.syncDocs(pluginName?)` — sync documentation for a plugin
|
|
36
|
-
- `.getSources()` — list available knowledge sources
|
|
68
|
+
或是使用 `postinstall` 脚本（已在 package.json 中配置）。
|
|
37
69
|
|
|
38
70
|
## License
|
|
39
71
|
|
package/dist/fetcher.d.ts
CHANGED
|
@@ -1,2 +1,5 @@
|
|
|
1
|
+
export declare function closeBrowser(): Promise<void>;
|
|
1
2
|
export declare function fetchHtml(url: string): Promise<string>;
|
|
3
|
+
export declare function fetchHtmlBrowser(url: string): Promise<string>;
|
|
4
|
+
export declare function fetchHtmlSmart(url: string): Promise<string>;
|
|
2
5
|
//# sourceMappingURL=fetcher.d.ts.map
|
package/dist/fetcher.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":"AAaA,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAKlD;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA8B5D;AAED,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAmBnE;AAeD,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMjE"}
|
package/dist/fetcher.js
CHANGED
|
@@ -1,4 +1,18 @@
|
|
|
1
|
-
|
|
1
|
+
import { chromium } from "playwright";
|
|
2
|
+
const USER_AGENT = "DevFlow-KnowledgeSync/0.3 (Node.js; Playwright)";
|
|
3
|
+
let _browser = null;
|
|
4
|
+
async function getBrowser() {
|
|
5
|
+
if (!_browser || !_browser.isConnected()) {
|
|
6
|
+
_browser = await chromium.launch({ headless: true });
|
|
7
|
+
}
|
|
8
|
+
return _browser;
|
|
9
|
+
}
|
|
10
|
+
export async function closeBrowser() {
|
|
11
|
+
if (_browser) {
|
|
12
|
+
await _browser.close().catch(() => { });
|
|
13
|
+
_browser = null;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
2
16
|
export async function fetchHtml(url) {
|
|
3
17
|
let lastError = null;
|
|
4
18
|
for (let attempt = 0; attempt <= 2; attempt++) {
|
|
@@ -20,11 +34,48 @@ export async function fetchHtml(url) {
|
|
|
20
34
|
catch (err) {
|
|
21
35
|
lastError = err;
|
|
22
36
|
if (attempt < 2) {
|
|
23
|
-
// Exponential backoff: 2s, 4s
|
|
24
37
|
await new Promise(r => setTimeout(r, 2000 * (attempt + 1)));
|
|
25
38
|
}
|
|
26
39
|
}
|
|
27
40
|
}
|
|
28
41
|
throw lastError ?? new Error(`Failed to fetch "${url}": unknown error`);
|
|
29
42
|
}
|
|
43
|
+
export async function fetchHtmlBrowser(url) {
|
|
44
|
+
const browser = await getBrowser();
|
|
45
|
+
const context = await browser.newContext({
|
|
46
|
+
userAgent: USER_AGENT,
|
|
47
|
+
});
|
|
48
|
+
const page = await context.newPage();
|
|
49
|
+
try {
|
|
50
|
+
await page.goto(url, {
|
|
51
|
+
waitUntil: "domcontentloaded",
|
|
52
|
+
timeout: 30_000,
|
|
53
|
+
});
|
|
54
|
+
// Let SPA frameworks render (React/Vue hydration usually <3s)
|
|
55
|
+
await page.waitForTimeout(4_000);
|
|
56
|
+
const html = await page.content();
|
|
57
|
+
return html;
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
await context.close().catch(() => { });
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function extractVisibleText(html) {
|
|
64
|
+
const text = html
|
|
65
|
+
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
|
|
66
|
+
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
|
|
67
|
+
.replace(/<[^>]+>/g, " ")
|
|
68
|
+
.replace(/ /g, " ")
|
|
69
|
+
.replace(/\s+/g, " ")
|
|
70
|
+
.trim();
|
|
71
|
+
return text.length;
|
|
72
|
+
}
|
|
73
|
+
const MIN_VISIBLE_TEXT = 500;
|
|
74
|
+
export async function fetchHtmlSmart(url) {
|
|
75
|
+
const html = await fetchHtml(url);
|
|
76
|
+
if (extractVisibleText(html) >= MIN_VISIBLE_TEXT) {
|
|
77
|
+
return html;
|
|
78
|
+
}
|
|
79
|
+
return fetchHtmlBrowser(url);
|
|
80
|
+
}
|
|
30
81
|
//# sourceMappingURL=fetcher.js.map
|
package/dist/fetcher.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,GAAG,
|
|
1
|
+
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../src/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAW,MAAM,YAAY,CAAC;AAE/C,MAAM,UAAU,GAAG,iDAAiD,CAAC;AAErE,IAAI,QAAQ,GAAmB,IAAI,CAAC;AAEpC,KAAK,UAAU,UAAU;IACvB,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;QACzC,QAAQ,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACvC,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW;IACzC,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC3B,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE;gBACrC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACpC,CAAC,CAAC;YAEH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACvB,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;gBACxE,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;gBACzD,SAAS;YACX,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,MAAM,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/E,CAAC;YAED,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAY,CAAC;YACzB,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,oBAAoB,GAAG,kBAAkB,CAAC,CAAC;AAC1E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAChD,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;QACvC,SAAS,EAAE,UAAU;KACtB,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAErC,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACnB,SAAS,EAAE,kBAAkB;YAC7B,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;QACH,8DAA8D;QAC9D,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,IAAI;SACd,OAAO,CAAC,mCAAmC,EAAE,EAAE,CAAC;SAChD,OAAO,CAAC,iCAAiC,EAAE,EAAE,CAAC;SAC9C,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;IACV,OAAO,IAAI,CAAC,MAAM,CAAC;AACrB,CAAC;AAED,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW;IAC9C,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,gBAAgB,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAC/B,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { KnowledgeEngine } from "./knowledge-engine.js";
|
|
2
|
-
export { fetchHtml } from "./fetcher.js";
|
|
2
|
+
export { fetchHtml, fetchHtmlBrowser, fetchHtmlSmart, closeBrowser } from "./fetcher.js";
|
|
3
3
|
export { htmlToMarkdown, sliceByHeadings } from "./markdown-converter.js";
|
|
4
4
|
export { bm25Search } from "./bm25.js";
|
|
5
5
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { KnowledgeEngine } from "./knowledge-engine.js";
|
|
2
|
-
export { fetchHtml } from "./fetcher.js";
|
|
2
|
+
export { fetchHtml, fetchHtmlBrowser, fetchHtmlSmart, closeBrowser } from "./fetcher.js";
|
|
3
3
|
export { htmlToMarkdown, sliceByHeadings } from "./markdown-converter.js";
|
|
4
4
|
export { bm25Search } from "./bm25.js";
|
|
5
5
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge-engine.d.ts","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,UAAU,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"knowledge-engine.d.ts","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,UAAU,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AA0B5H,qBAAa,eAAe;IAC1B,OAAO,CAAC,WAAW,CAAS;gBAEhB,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE;IAIzC,OAAO,KAAK,YAAY,GAEvB;IAED,OAAO,CAAC,SAAS;IAIX,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IA8BtD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IA8EpD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAsEzE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAOtE,UAAU,IAAI,OAAO,CAAC,qBAAqB,EAAE,CAAC;CAiCrD"}
|
package/dist/knowledge-engine.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "fs";
|
|
2
2
|
import { join } from "path";
|
|
3
|
-
import {
|
|
3
|
+
import { fetchHtmlSmart, fetchHtmlBrowser, closeBrowser } from "./fetcher.js";
|
|
4
4
|
import { htmlToMarkdown, sliceByHeadings } from "./markdown-converter.js";
|
|
5
5
|
import { bm25Search } from "./bm25.js";
|
|
6
6
|
export class KnowledgeEngine {
|
|
@@ -29,6 +29,7 @@ export class KnowledgeEngine {
|
|
|
29
29
|
pluginName: source.pluginName || existing.pluginName,
|
|
30
30
|
version: source.version || existing.version,
|
|
31
31
|
sourceUrl: existing.sourceUrl,
|
|
32
|
+
renderJs: source.renderJs ?? existing.renderJs ?? false,
|
|
32
33
|
}, null, 2), "utf-8");
|
|
33
34
|
}
|
|
34
35
|
else {
|
|
@@ -38,6 +39,7 @@ export class KnowledgeEngine {
|
|
|
38
39
|
chunkCount: 0,
|
|
39
40
|
pluginName: source.pluginName,
|
|
40
41
|
version: source.version,
|
|
42
|
+
renderJs: source.renderJs ?? false,
|
|
41
43
|
}, null, 2), "utf-8");
|
|
42
44
|
}
|
|
43
45
|
}
|
|
@@ -69,7 +71,9 @@ export class KnowledgeEngine {
|
|
|
69
71
|
const errors = [];
|
|
70
72
|
let added = 0;
|
|
71
73
|
try {
|
|
72
|
-
const html =
|
|
74
|
+
const html = manifest.renderJs
|
|
75
|
+
? await fetchHtmlBrowser(url)
|
|
76
|
+
: await fetchHtmlSmart(url);
|
|
73
77
|
const markdown = htmlToMarkdown(html);
|
|
74
78
|
const chunks = sliceByHeadings(markdown);
|
|
75
79
|
const chunkFiles = chunks.map((chunk, i) => ({
|
|
@@ -91,6 +95,7 @@ export class KnowledgeEngine {
|
|
|
91
95
|
pluginName: manifest.pluginName,
|
|
92
96
|
version: manifest.version,
|
|
93
97
|
sourceUrl: url,
|
|
98
|
+
renderJs: manifest.renderJs ?? false,
|
|
94
99
|
}, null, 2), "utf-8");
|
|
95
100
|
}
|
|
96
101
|
catch (err) {
|
|
@@ -98,6 +103,7 @@ export class KnowledgeEngine {
|
|
|
98
103
|
}
|
|
99
104
|
results.push({ source: url, added, updated: 0, errors });
|
|
100
105
|
}
|
|
106
|
+
await closeBrowser();
|
|
101
107
|
return results;
|
|
102
108
|
}
|
|
103
109
|
async search(query, options) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge-engine.js","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACrF,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"knowledge-engine.js","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACrF,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAqBvC,MAAM,OAAO,eAAe;IAClB,WAAW,CAAS;IAE5B,YAAY,IAA6B;QACvC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IACtC,CAAC;IAED,IAAY,YAAY;QACtB,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;IACzD,CAAC;IAEO,SAAS,CAAC,UAAkB;QAClC,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAuB;QAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAEhD,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAiB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;YAC/E,6CAA6C;YAC7C,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpE,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC;gBACzC,IAAI,EAAE,UAAU;gBAChB,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU;gBACpD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO;gBAC3C,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,IAAI,KAAK;aACxD,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC;gBACzC,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK;aACnC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,UAAmB;QAChC,MAAM,IAAI,GAAG,UAAU,IAAI,EAAE,CAAC;QAE9B,qDAAqD;QACrD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,UAAU,GAAiB,EAAE,CAAC;YACpC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC;gBAAE,OAAO,EAAE,CAAC;YAE9C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;iBACjE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;iBAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAEpB,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAC7C,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,CAAC;YACD,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QAEzC,MAAM,QAAQ,GAAiB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAa,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;QAE3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ;oBAC5B,CAAC,CAAC,MAAM,gBAAgB,CAAC,GAAG,CAAC;oBAC7B,CAAC,CAAC,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;gBACtC,MAAM,MAAM,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;gBAEzC,MAAM,UAAU,GAAgB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;oBACxD,EAAE,EAAE,SAAS,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;oBAC7C,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,OAAO,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ;oBAC9B,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,SAAS,EAAE,GAAG;oBACd,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC,CAAC,CAAC;gBAEJ,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;oBAC5B,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBAClF,CAAC;gBAED,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC;gBAE1B,aAAa,CACX,YAAY,EACZ,IAAI,CAAC,SAAS,CAAC;oBACb,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,UAAU,EAAE,UAAU,CAAC,MAAM;oBAC7B,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;oBACzB,SAAS,EAAE,GAAG;oBACd,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,KAAK;iBACrC,EAAE,IAAI,EAAE,CAAC,CAAC,EACX,OAAO,CACR,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAE,GAAa,CAAC,OAAO,CAAC,CAAC;YACtC,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,YAAY,EAAE,CAAC;QACrB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAuB;QACjD,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAEjC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QAE9C,MAAM,SAAS,GAGV,EAAE,CAAC;QAER,MAAM,UAAU,GAAG,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aAClF,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEpB,KAAK,MAAM,UAAU,IAAI,UAAU,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YACvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAE/B,6BAA6B;YAC7B,IAAI,OAA2B,CAAC;YAChC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;YAChD,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,CAAC,GAAiB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;oBACxE,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;gBACtB,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YAC1B,CAAC;YAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACH,MAAM,KAAK,GAAc,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;oBAC5E,SAAS,CAAC,IAAI,CAAC;wBACb,EAAE,EAAE,GAAG,UAAU,IAAI,KAAK,CAAC,EAAE,EAAE;wBAC/B,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;wBACtB,SAAS,EAAE,KAAK,CAAC,SAAS;wBAC1B,UAAU;wBACV,OAAO;qBACR,CAAC,CAAC;gBACL,CAAC;gBAAC,MAAM,CAAC;oBACP,uBAAuB;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,MAAM,EAAE,CAAC,CAAC,UAAU;gBACpB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBAChC,GAAG,EAAE,CAAC,CAAC,SAAS;gBAChB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,SAAS,EAAE,EAAE;aACd,CAAC,CAAC,CAAC;QACN,CAAC;QAED,OAAO,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU,IAAI,EAAE;YAC5D,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,GAAG,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS;YAClD,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO;YACpD,SAAS,EAAE,EAAE;SACd,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAc,EAAE,KAAc;QACzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACtF,OAAO,KAAK,IAAI,IAAI,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QAE9C,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aACvE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEpB,OAAO,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YAC3B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC;YAC7C,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,IAAI,GAAa,EAAE,CAAC;YACxB,IAAI,UAAU,GAAG,EAAE,CAAC;YACpB,IAAI,OAAO,GAAG,EAAE,CAAC;YAEjB,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAiB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC/E,UAAU,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;gBACtC,QAAQ,GAAG,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;gBACnC,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC3B,UAAU,GAAG,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;gBACvC,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;YACnC,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE;gBAC3C,MAAM,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,QAAiB,CAAC,CAAC,CAAC,MAAe;gBAC5D,YAAY,EAAE,QAAQ,IAAI,SAAS;gBACnC,UAAU,EAAE,UAAU,IAAI,SAAS;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devflow-tools/knowledge-engine",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.11",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -14,10 +14,12 @@
|
|
|
14
14
|
"build": "tsc -b",
|
|
15
15
|
"test": "node ../../node_modules/.bin/vitest run",
|
|
16
16
|
"typecheck": "tsc --noEmit",
|
|
17
|
-
"clean": "rm -rf dist"
|
|
17
|
+
"clean": "rm -rf dist",
|
|
18
|
+
"postinstall": "playwright install chromium || true"
|
|
18
19
|
},
|
|
19
20
|
"dependencies": {
|
|
20
|
-
"@devflow-tools/sdk": "^0.8.
|
|
21
|
+
"@devflow-tools/sdk": "^0.8.11",
|
|
22
|
+
"playwright": "^1.52.0",
|
|
21
23
|
"turndown": "^7.2.4"
|
|
22
24
|
},
|
|
23
25
|
"devDependencies": {
|
|
@@ -28,5 +30,5 @@
|
|
|
28
30
|
"files": [
|
|
29
31
|
"dist"
|
|
30
32
|
],
|
|
31
|
-
"gitHead": "
|
|
33
|
+
"gitHead": "a0a0b61377aa88ebbd380f50a5d532f63175ef45"
|
|
32
34
|
}
|