deep-read-summarize 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +3 -1
- package/docs/RELEASE.md +17 -2
- package/lib/cache.js +28 -3
- package/package.json +5 -2
- package/parsers/_prompt.js +44 -0
- package/parsers/book.js +17 -19
- package/parsers/paper.js +17 -18
- package/parsers/video.js +18 -19
- package/parsers/web.js +16 -17
- package/scripts/lint.js +8 -3
- package/scripts/security-check.js +42 -20
- package/tests/cache.test.js +14 -0
- package/tests/run-tests.js +30 -0
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,23 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
The format is based on [Keep a Changelog](https://keepachangelog.com/).
|
|
5
5
|
|
|
6
|
+
## [0.3.0] — 2026-08-21
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
- 幂等缓存 URL 规范化(normalizeInput):host 大小写 / 默认端口 / fragment / 尾部斜杠归一,同一 URL 不同写法共享指纹
|
|
10
|
+
- 安全扫描覆盖 .env.example 与 forward-slash 用户路径;导出 scanDir 供测试调用
|
|
11
|
+
- 测试 17 → 21:parser 模板回归、URL 规范化、安全扫描正/负样本
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
- parsers 四解析器去重:抽取共享提示词模板 parsers/_prompt.js(buildPrompt 输出逐字不变,经新旧对比验证)
|
|
15
|
+
- lint 排除 .tmp*/dist/coverage 等临时目录
|
|
16
|
+
- CI 增加 Node 18/20/22 版本矩阵
|
|
17
|
+
- package.json 增加 publishConfig.access=public(显式公开发布)
|
|
18
|
+
- docs/RELEASE.md 增加 npm 发布流程与版本速览;README 开发节补充发布门禁说明
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
- security-check 对测试样本误报(样本改为拼接构造,源码不含完整密钥模式)
|
|
22
|
+
|
|
6
23
|
## [0.2.0] — 2026-08-21
|
|
7
24
|
|
|
8
25
|
### Added
|
package/README.md
CHANGED
|
@@ -198,10 +198,12 @@ parsers: book, paper, video, web
|
|
|
198
198
|
```bash
|
|
199
199
|
npm install
|
|
200
200
|
npm run lint # node --check 全部 JS
|
|
201
|
-
npm test # fixture
|
|
201
|
+
npm test # fixture 测试(21 项,全部离线)
|
|
202
202
|
npm run validate # 发布前验证(含安全检查)
|
|
203
203
|
```
|
|
204
204
|
|
|
205
|
+
npm publish 会自动先跑 prepublishOnly(测试 + lint + 安全检查),任何一项失败都不会发布。
|
|
206
|
+
|
|
205
207
|
改动说明见 CHANGELOG.md,贡献规范见 CONTRIBUTING.md。
|
|
206
208
|
|
|
207
209
|
---
|
package/docs/RELEASE.md
CHANGED
|
@@ -32,12 +32,27 @@
|
|
|
32
32
|
|
|
33
33
|
---
|
|
34
34
|
|
|
35
|
+
## npm 发布流程(v0.2.0 起)
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
cd D:\APP\deep-read-summarize
|
|
39
|
+
npm run lint && npm test && npm run validate # 先本地全绿
|
|
40
|
+
npm pkg fix # 规范化 package.json(npm 建议)
|
|
41
|
+
npm publish # prepublishOnly 自动跑门禁(测试+lint+安全)
|
|
42
|
+
npm view deep-read-summarize --prefer-online # 验证线上(注意:发布后首次查询可能 404,索引有延迟)
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
- token 存在用户级 ~/.npmrc(//registry.npmjs.org/:_authToken=...),npm whoami 可验证
|
|
46
|
+
- 本沙箱环境 npm 默认缓存目录可能被拒(EPERM),用 npm publish --cache <工作区内目录>
|
|
47
|
+
- 版本号:功能增强 → minor(0.x → 0.y);修复 → patch;不兼容 → major
|
|
48
|
+
|
|
35
49
|
## 当前版本速览
|
|
36
50
|
|
|
37
51
|
| 版本 | 状态 | 说明 |
|
|
38
52
|
|------|------|------|
|
|
39
|
-
| v0.1.
|
|
40
|
-
|
|
|
53
|
+
| v0.1.0/v0.1.1 | git tag | 早期迭代,未上 npm |
|
|
54
|
+
| v0.2.x | npm 已发布 | 首个 npm 版本,含发布门禁 |
|
|
55
|
+
| v1.0.0 | 计划 | API 稳定后发布 |
|
|
41
56
|
|
|
42
57
|
> 早期迭代用 0.x 表达不稳定;API 稳定后再发 1.0.0。
|
|
43
58
|
> Tag 发布后保持稳定,出问题发修复版,不要反复改同一个 Tag。
|
package/lib/cache.js
CHANGED
|
@@ -16,9 +16,34 @@ const fs = require("fs");
|
|
|
16
16
|
const path = require("path");
|
|
17
17
|
const crypto = require("crypto");
|
|
18
18
|
|
|
19
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* 规范化输入:对 http(s) URL 做归一化(host 小写、去默认端口、去 fragment、
|
|
21
|
+
* 去空 query 的 "?"、去尾部斜杠),避免同一 URL 的不同写法产生不同指纹;
|
|
22
|
+
* 非 URL(本地路径/纯文本)原样返回。
|
|
23
|
+
*/
|
|
24
|
+
function normalizeInput(input) {
|
|
25
|
+
const s = typeof input === "string" ? input.trim() : "";
|
|
26
|
+
if (!/^https?:\/\//i.test(s)) return s;
|
|
27
|
+
try {
|
|
28
|
+
const url = new URL(s);
|
|
29
|
+
url.hash = "";
|
|
30
|
+
url.hostname = url.hostname.toLowerCase();
|
|
31
|
+
if ((url.protocol === "http:" && url.port === "80") || (url.protocol === "https:" && url.port === "443")) {
|
|
32
|
+
url.port = "";
|
|
33
|
+
}
|
|
34
|
+
if (url.search === "?") url.search = "";
|
|
35
|
+
if (url.pathname.length > 1 && url.pathname.endsWith("/")) {
|
|
36
|
+
url.pathname = url.pathname.slice(0, -1);
|
|
37
|
+
}
|
|
38
|
+
return url.toString();
|
|
39
|
+
} catch (e) {
|
|
40
|
+
return s; // 非法 URL 原样处理
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** 计算输入指纹:URL 规范化后 hash;本地文件加大小 + 修改时间 */
|
|
20
45
|
function fingerprint(input, fileMeta) {
|
|
21
|
-
const base =
|
|
46
|
+
const base = normalizeInput(input);
|
|
22
47
|
if (!base) return null;
|
|
23
48
|
const metaStr = fileMeta
|
|
24
49
|
? "|" + (fileMeta.size || "") + "|" + (fileMeta.mtimeMs || "")
|
|
@@ -59,4 +84,4 @@ function markProcessed(cacheFile, key, outputPath, meta) {
|
|
|
59
84
|
return true;
|
|
60
85
|
}
|
|
61
86
|
|
|
62
|
-
module.exports = { fingerprint, readCache, hasProcessed, markProcessed };
|
|
87
|
+
module.exports = { fingerprint, normalizeInput, readCache, hasProcessed, markProcessed };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "deep-read-summarize",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Deep reading & summarization for DSH: books/papers/videos/web → structured Obsidian notes. Plugin parsers, MapReduce deep-read, JSON Schema output, idempotent cache.",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "index.js",
|
|
@@ -56,11 +56,14 @@
|
|
|
56
56
|
"parsers"
|
|
57
57
|
],
|
|
58
58
|
"license": "MIT",
|
|
59
|
+
"publishConfig": {
|
|
60
|
+
"access": "public"
|
|
61
|
+
},
|
|
59
62
|
"engines": {
|
|
60
63
|
"node": ">=18"
|
|
61
64
|
},
|
|
62
65
|
"repository": {
|
|
63
66
|
"type": "git",
|
|
64
|
-
"url": "https://github.com/PensiveFei/deep-read-summarize.git"
|
|
67
|
+
"url": "git+https://github.com/PensiveFei/deep-read-summarize.git"
|
|
65
68
|
}
|
|
66
69
|
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// parsers/_prompt.js — 共享「获取+分块」提示词模板(内部模块)
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 四个内置解析器共用同一提示词骨架,各自只提供差异化配置:
|
|
5
|
+
// - label/kindLabel:类型名称(中文 / 中英混合)
|
|
6
|
+
// - steps:获取文本的具体步骤(不含分块步骤,模板自动编号)
|
|
7
|
+
// - chunkRule:分块规则描述
|
|
8
|
+
// - jsonExample:输出 JSON 示例(含 kind 字段)
|
|
9
|
+
// 保持各解析器 buildPrompt(input, opts) 插件接口不变;fork 时仍可整体重写。
|
|
10
|
+
// ============================================================================
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* 构建「获取+分块」提示词。
|
|
14
|
+
* @param {object} cfg
|
|
15
|
+
* @param {string} cfg.input 输入(URL 或本地路径)
|
|
16
|
+
* @param {object} [cfg.opts] 选项(tempDir/tempFile/maxChunks)
|
|
17
|
+
* @param {string} cfg.fetchTarget 获取对象描述(如「书籍全文」「视频字幕/转写」)
|
|
18
|
+
* @param {string} cfg.kindLabel 内容类型描述(如「书籍(book)」)
|
|
19
|
+
* @param {string[]} cfg.steps 获取步骤(不含分块步骤;模板统一编号并追加分块步骤)
|
|
20
|
+
* @param {string} cfg.chunkRule 分块规则(如「章节(或章节内小节)」)
|
|
21
|
+
* @param {string} cfg.jsonExample 输出 JSON 示例
|
|
22
|
+
* @returns {string} 完整提示词
|
|
23
|
+
*/
|
|
24
|
+
function buildFetchPrompt(cfg) {
|
|
25
|
+
const opts = cfg.opts || {};
|
|
26
|
+
const tempDir = opts.tempDir || "./.tmp";
|
|
27
|
+
const tempFile = tempDir + "\\input.txt";
|
|
28
|
+
const maxChunks = opts.maxChunks || 6;
|
|
29
|
+
const steps = (cfg.steps || []).map(function (s, i) { return (i + 1) + ". " + s; });
|
|
30
|
+
return [
|
|
31
|
+
"你的任务:获取" + cfg.fetchTarget + "并保存为纯文本,然后给出分块计划。",
|
|
32
|
+
"内容类型:" + cfg.kindLabel,
|
|
33
|
+
"输入来源:" + cfg.input,
|
|
34
|
+
"目标文件:" + tempFile + "(先创建目录 " + tempDir + ",用 write 工具写入)",
|
|
35
|
+
"步骤:"
|
|
36
|
+
].concat(steps, [
|
|
37
|
+
(steps.length + 1) + ". 分块:按" + cfg.chunkRule + "切分,每块约 100-200 行或 4000-6000 字,块数不超过 " + maxChunks + " 块;每块给 id/topic/startLine/endLine。",
|
|
38
|
+
"输出 JSON:",
|
|
39
|
+
cfg.jsonExample,
|
|
40
|
+
"若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
|
|
41
|
+
]).join("\n");
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
module.exports = { buildFetchPrompt };
|
package/parsers/book.js
CHANGED
|
@@ -3,31 +3,29 @@
|
|
|
3
3
|
// ============================================================================
|
|
4
4
|
// 独立解析器插件:处理书籍输入。fork 后替换此文件即可支持自定义格式。
|
|
5
5
|
// 接口约定:module.exports = { name, types, buildPrompt(input, opts) }
|
|
6
|
+
// 提示词骨架复用 parsers/_prompt.js(去重),此处只提供差异化配置。
|
|
6
7
|
// ============================================================================
|
|
7
8
|
|
|
9
|
+
const prompt = require("./_prompt");
|
|
10
|
+
|
|
8
11
|
module.exports = {
|
|
9
12
|
name: "book",
|
|
10
13
|
types: ["book"],
|
|
11
14
|
description: "书籍/长篇著作解析器(PDF/EPUB/MOBI → 文本 → 逻辑分块)",
|
|
12
15
|
|
|
13
|
-
// 构建「获取+分块」提示词
|
|
14
16
|
buildPrompt: function (input, opts) {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
"
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
"
|
|
27
|
-
|
|
28
|
-
"输出 JSON:",
|
|
29
|
-
'{ "kind": "book", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "章节主题", "startLine": 1, "endLine": 200 } ] }',
|
|
30
|
-
"若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
|
|
31
|
-
].join("\n");
|
|
17
|
+
return prompt.buildFetchPrompt({
|
|
18
|
+
input: input,
|
|
19
|
+
opts: opts,
|
|
20
|
+
fetchTarget: "书籍全文",
|
|
21
|
+
kindLabel: "书籍(book)",
|
|
22
|
+
steps: [
|
|
23
|
+
"获取全文:PDF→用 read 或 pwsh 调 pdftotext(若为扫描版报告需 OCR);EPUB/MOBI→解包或转文本;纯文本→直接读取。",
|
|
24
|
+
"全文写入目标文件(不截断、不加评注),用 read 确认总行数。",
|
|
25
|
+
"识别书籍结构:目录/序言/各章节标题及其行号范围。"
|
|
26
|
+
],
|
|
27
|
+
chunkRule: "章节(或章节内小节)",
|
|
28
|
+
jsonExample: '{ "kind": "book", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "章节主题", "startLine": 1, "endLine": 200 } ] }'
|
|
29
|
+
});
|
|
32
30
|
}
|
|
33
|
-
};
|
|
31
|
+
};
|
package/parsers/paper.js
CHANGED
|
@@ -2,30 +2,29 @@
|
|
|
2
2
|
// Parser plugin: paper (arXiv/PDF/HTML academic paper)
|
|
3
3
|
// ============================================================================
|
|
4
4
|
// 独立解析器插件:处理学术论文输入。fork 后替换此文件即可支持自定义来源。
|
|
5
|
+
// 提示词骨架复用 parsers/_prompt.js(去重),此处只提供差异化配置。
|
|
5
6
|
// ============================================================================
|
|
6
7
|
|
|
8
|
+
const prompt = require("./_prompt");
|
|
9
|
+
|
|
7
10
|
module.exports = {
|
|
8
11
|
name: "paper",
|
|
9
12
|
types: ["paper"],
|
|
10
13
|
description: "学术论文解析器(arXiv/期刊/PDF/HTML)",
|
|
11
14
|
|
|
12
15
|
buildPrompt: function (input, opts) {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
"
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
"
|
|
25
|
-
|
|
26
|
-
"输出 JSON:",
|
|
27
|
-
'{ "kind": "paper", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "Introduction", "startLine": 1, "endLine": 200 } ] }',
|
|
28
|
-
"若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
|
|
29
|
-
].join("\n");
|
|
16
|
+
return prompt.buildFetchPrompt({
|
|
17
|
+
input: input,
|
|
18
|
+
opts: opts,
|
|
19
|
+
fetchTarget: "论文全文",
|
|
20
|
+
kindLabel: "学术论文(paper)",
|
|
21
|
+
steps: [
|
|
22
|
+
"获取全文:arXiv→访问 abs 页取 PDF/HTML 全文;PDF→read 或 pdftotext;HTML→提取正文(去导航/广告)。",
|
|
23
|
+
"全文写入目标文件(不截断、不加评注),用 read 确认总行数。",
|
|
24
|
+
"识别论文结构:Abstract/Introduction/Methods/Results/Discussion/Conclusion/Appendix 及各表图行号范围。"
|
|
25
|
+
],
|
|
26
|
+
chunkRule: "论文章节",
|
|
27
|
+
jsonExample: '{ "kind": "paper", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "Introduction", "startLine": 1, "endLine": 200 } ] }'
|
|
28
|
+
});
|
|
30
29
|
}
|
|
31
|
-
};
|
|
30
|
+
};
|
package/parsers/video.js
CHANGED
|
@@ -2,31 +2,30 @@
|
|
|
2
2
|
// Parser plugin: video (YouTube/Bilibili subtitles & transcripts)
|
|
3
3
|
// ============================================================================
|
|
4
4
|
// 独立解析器插件:处理视频输入。优先 yt-dlp 自动抓字幕,失败降级人工转写。
|
|
5
|
+
// 提示词骨架复用 parsers/_prompt.js(去重),此处只提供差异化配置。
|
|
5
6
|
// ============================================================================
|
|
6
7
|
|
|
8
|
+
const prompt = require("./_prompt");
|
|
9
|
+
|
|
7
10
|
module.exports = {
|
|
8
11
|
name: "video",
|
|
9
12
|
types: ["video"],
|
|
10
13
|
description: "视频解析器(YouTube/B站/播客 → 字幕 → 文本)",
|
|
11
14
|
|
|
12
15
|
buildPrompt: function (input, opts) {
|
|
13
|
-
const tempDir = opts.tempDir || "./.tmp";
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
"
|
|
18
|
-
"
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
"
|
|
26
|
-
|
|
27
|
-
"输出 JSON:",
|
|
28
|
-
'{ "kind": "video", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "主题段", "startLine": 1, "endLine": 200 } ] }',
|
|
29
|
-
"若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
|
|
30
|
-
].join("\n");
|
|
16
|
+
const tempDir = (opts && opts.tempDir) || "./.tmp";
|
|
17
|
+
return prompt.buildFetchPrompt({
|
|
18
|
+
input: input,
|
|
19
|
+
opts: opts,
|
|
20
|
+
fetchTarget: "视频字幕/转写",
|
|
21
|
+
kindLabel: "视频(video)",
|
|
22
|
+
steps: [
|
|
23
|
+
"用 pwsh 运行 yt-dlp 抓字幕,命令示例:\n yt-dlp --skip-download --write-auto-sub --write-subs --sub-langs 'zh,en' --sub-format vtt/srt -o '" + tempDir + "/sub.%(ext)s' <URL>",
|
|
24
|
+
"读取字幕文件(vtt/srt),去除时间戳与 HTML 标签,转为纯文本写入目标文件。",
|
|
25
|
+
"若 yt-dlp 不可用或抓取失败:输出 { \"saved\": false, \"message\": \"需要人工提供转写文本\" }。"
|
|
26
|
+
],
|
|
27
|
+
chunkRule: "主题脉络(而非时间)",
|
|
28
|
+
jsonExample: '{ "kind": "video", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "主题段", "startLine": 1, "endLine": 200 } ] }'
|
|
29
|
+
});
|
|
31
30
|
}
|
|
32
|
-
};
|
|
31
|
+
};
|
package/parsers/web.js
CHANGED
|
@@ -2,29 +2,28 @@
|
|
|
2
2
|
// Parser plugin: web (generic web page / article)
|
|
3
3
|
// ============================================================================
|
|
4
4
|
// 独立解析器插件:处理网页/文章输入(通用 fallback)。
|
|
5
|
+
// 提示词骨架复用 parsers/_prompt.js(去重),此处只提供差异化配置。
|
|
5
6
|
// ============================================================================
|
|
6
7
|
|
|
8
|
+
const prompt = require("./_prompt");
|
|
9
|
+
|
|
7
10
|
module.exports = {
|
|
8
11
|
name: "web",
|
|
9
12
|
types: ["web"],
|
|
10
13
|
description: "网页/文章解析器(正文提取 → 分块)",
|
|
11
14
|
|
|
12
15
|
buildPrompt: function (input, opts) {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
"
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
"
|
|
24
|
-
|
|
25
|
-
"输出 JSON:",
|
|
26
|
-
'{ "kind": "web", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "主题", "startLine": 1, "endLine": 200 } ] }',
|
|
27
|
-
"若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
|
|
28
|
-
].join("\n");
|
|
16
|
+
return prompt.buildFetchPrompt({
|
|
17
|
+
input: input,
|
|
18
|
+
opts: opts,
|
|
19
|
+
fetchTarget: "网页/文章正文",
|
|
20
|
+
kindLabel: "网页/文章(web)",
|
|
21
|
+
steps: [
|
|
22
|
+
"访问链接获取正文文本(去除导航、广告、页脚等噪音),或直接读取本地文本文件。",
|
|
23
|
+
"正文写入目标文件(不截断、不加评注),用 read 确认总行数。"
|
|
24
|
+
],
|
|
25
|
+
chunkRule: "逻辑主题",
|
|
26
|
+
jsonExample: '{ "kind": "web", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "主题", "startLine": 1, "endLine": 200 } ] }'
|
|
27
|
+
});
|
|
29
28
|
}
|
|
30
|
-
};
|
|
29
|
+
};
|
package/scripts/lint.js
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
// scripts/lint.js — syntax-check all JS files (node --check)
|
|
2
|
+
// 跳过 node_modules / 隐藏目录(.git, .tmp* 等)/ 构建与临时目录。
|
|
2
3
|
const { execSync } = require('child_process');
|
|
3
4
|
const fs = require('fs');
|
|
4
5
|
const path = require('path');
|
|
5
6
|
|
|
7
|
+
const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'coverage', '.nyc_output', '_deepread_temp']);
|
|
8
|
+
|
|
6
9
|
function findJs(dir, acc = []) {
|
|
7
10
|
for (const f of fs.readdirSync(dir)) {
|
|
11
|
+
if (f.startsWith('.tmp')) continue; // 临时目录(.tmp* / .tmp-old-*)
|
|
8
12
|
const p = path.join(dir, f);
|
|
9
13
|
const st = fs.statSync(p);
|
|
10
|
-
if (st.isDirectory()) {
|
|
11
|
-
|
|
14
|
+
if (st.isDirectory()) {
|
|
15
|
+
if (!SKIP_DIRS.has(f)) findJs(p, acc);
|
|
16
|
+
} else if (f.endsWith('.js')) acc.push(p);
|
|
12
17
|
}
|
|
13
18
|
return acc;
|
|
14
19
|
}
|
|
@@ -20,4 +25,4 @@ for (const f of files) {
|
|
|
20
25
|
catch (e) { console.error('LINT FAIL:', f); failed++; }
|
|
21
26
|
}
|
|
22
27
|
if (failed) { console.error(failed + ' file(s) failed'); process.exit(1); }
|
|
23
|
-
console.log('Lint OK:', files.length, 'files');
|
|
28
|
+
console.log('Lint OK:', files.length, 'files');
|
|
@@ -1,32 +1,54 @@
|
|
|
1
|
+
// scripts/security-check.js — 密钥/隐私/路径扫描(可被测试调用)
|
|
2
|
+
// 扫描仓库内文本文件,检查常见密钥模式与本地用户路径泄露。
|
|
3
|
+
// 直接运行:node scripts/security-check.js(exit 1 = 发现问题)
|
|
4
|
+
// 测试调用:const { scanDir } = require('./scripts/security-check'); scanDir(dir);
|
|
1
5
|
const fs = require('fs');
|
|
2
6
|
const path = require('path');
|
|
3
7
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
if (st.isDirectory()) { if (f !== 'node_modules' && f !== '.git') walk(p, acc); }
|
|
9
|
-
else if (/\.(js|md|yml|json|txt)$/.test(f)) acc.push(p);
|
|
10
|
-
}
|
|
11
|
-
return acc;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
const files = walk(process.cwd());
|
|
15
|
-
const issues = [];
|
|
16
|
-
const patterns = [
|
|
8
|
+
const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'coverage', '.nyc_output', '_deepread_temp']);
|
|
9
|
+
// 匹配文件后缀:js/md/yml/yaml/json/txt + .env.example(.example 结尾)
|
|
10
|
+
const FILE_RE = /\.(js|md|yml|yaml|json|txt|example)$/i;
|
|
11
|
+
const PATTERNS = [
|
|
17
12
|
[/api[_-]?key\s*[:=]\s*['"][^'"]+/i, 'API KEY pattern'],
|
|
18
13
|
[/sk-[A-Za-z0-9]{20,}/, 'OpenAI-style key'],
|
|
19
14
|
[/password\s*[:=]\s*['"][^'"]+/i, 'password pattern'],
|
|
20
|
-
[/C:\\Users\\[^\\"']+/i, 'Windows user path'],
|
|
15
|
+
[/C:\\Users\\[^\\"' ]+/i, 'Windows user path'],
|
|
16
|
+
[/C:\/Users\/[^"' ]+/i, 'Windows user path (forward slash)'],
|
|
21
17
|
[/AKIA[0-9A-Z]{16}/, 'AWS key']
|
|
22
18
|
];
|
|
23
19
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
20
|
+
function walk(dir, acc = []) {
|
|
21
|
+
let entries;
|
|
22
|
+
try { entries = fs.readdirSync(dir, { withFileTypes: true }); }
|
|
23
|
+
catch (e) { return acc; }
|
|
24
|
+
for (const ent of entries) {
|
|
25
|
+
if (ent.name.startsWith('.tmp')) continue; // 临时目录
|
|
26
|
+
const p = path.join(dir, ent.name);
|
|
27
|
+
if (ent.isDirectory()) {
|
|
28
|
+
if (!SKIP_DIRS.has(ent.name)) walk(p, acc);
|
|
29
|
+
} else if (FILE_RE.test(ent.name)) acc.push(p);
|
|
30
|
+
}
|
|
31
|
+
return acc;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** 扫描目录,返回问题列表(空数组 = 干净) */
|
|
35
|
+
function scanDir(dir) {
|
|
36
|
+
const files = walk(dir);
|
|
37
|
+
const issues = [];
|
|
38
|
+
for (const f of files) {
|
|
39
|
+
let c;
|
|
40
|
+
try { c = fs.readFileSync(f, 'utf8'); } catch (e) { continue; }
|
|
41
|
+
for (const [re, label] of PATTERNS) {
|
|
42
|
+
if (re.test(c)) issues.push(label + ' in ' + path.relative(dir, f));
|
|
43
|
+
}
|
|
28
44
|
}
|
|
45
|
+
return issues;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (require.main === module) {
|
|
49
|
+
const issues = scanDir(process.cwd());
|
|
50
|
+
if (issues.length === 0) { console.log('OK: no secrets/passwords/local user paths found'); }
|
|
51
|
+
else { issues.forEach(i => console.log('ISSUE: ' + i)); process.exit(1); }
|
|
29
52
|
}
|
|
30
53
|
|
|
31
|
-
|
|
32
|
-
else { issues.forEach(i => console.log('ISSUE: ' + i)); process.exit(1); }
|
|
54
|
+
module.exports = { scanDir, walk, PATTERNS };
|
package/tests/cache.test.js
CHANGED
|
@@ -27,6 +27,20 @@ test('fingerprint differs for different inputs', () => {
|
|
|
27
27
|
assert.notStrictEqual(a, b);
|
|
28
28
|
});
|
|
29
29
|
|
|
30
|
+
test('fingerprint normalizes URL variants', () => {
|
|
31
|
+
const a = cache.fingerprint('https://EXAMPLE.com/doc/');
|
|
32
|
+
const b = cache.fingerprint('https://example.com/doc#frag');
|
|
33
|
+
assert.strictEqual(a, b, 'URL variants (case/slash/fragment) should share fingerprint');
|
|
34
|
+
const c = cache.fingerprint('https://example.com/doc?x=1');
|
|
35
|
+
assert.notStrictEqual(a, c, 'different query should produce different fingerprint');
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test('normalizeInput keeps non-URL input and root path unchanged', () => {
|
|
39
|
+
assert.strictEqual(cache.normalizeInput('C:\\Users\\x\\doc.md'), 'C:\\Users\\x\\doc.md');
|
|
40
|
+
assert.strictEqual(cache.normalizeInput('https://example.com/'), 'https://example.com/');
|
|
41
|
+
assert.strictEqual(cache.normalizeInput(' '), '');
|
|
42
|
+
});
|
|
43
|
+
|
|
30
44
|
test('fingerprint includes file meta for local files', () => {
|
|
31
45
|
const f = path.join(tmp, 'sample.txt');
|
|
32
46
|
fs.mkdirSync(tmp, { recursive: true });
|
package/tests/run-tests.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// tests/run-tests.js — fixture-driven tests
|
|
2
2
|
const assert = require('assert');
|
|
3
3
|
const fs = require('fs');
|
|
4
|
+
const os = require('os');
|
|
4
5
|
const path = require('path');
|
|
5
6
|
|
|
6
7
|
let passed = 0, failed = 0;
|
|
@@ -33,6 +34,16 @@ test('every parser builds a non-empty prompt', () => {
|
|
|
33
34
|
}
|
|
34
35
|
});
|
|
35
36
|
|
|
37
|
+
test('parser prompts embed input, tempFile and maxChunks', () => {
|
|
38
|
+
for (const type of ['book', 'paper', 'video', 'web']) {
|
|
39
|
+
const p = parsers.resolve(type);
|
|
40
|
+
const prompt = p.buildPrompt('https://x.test/doc', { tempDir: '/t', maxChunks: 4 });
|
|
41
|
+
assert.ok(prompt.includes('https://x.test/doc'), type + ': input missing');
|
|
42
|
+
assert.ok(prompt.includes('/t\\input.txt') || prompt.includes('/t/input.txt'), type + ': tempFile missing');
|
|
43
|
+
assert.ok(prompt.includes('块数不超过 4 块'), type + ': maxChunks not applied');
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
|
|
36
47
|
// ---------- Test: schemas ----------
|
|
37
48
|
const schemas = require('../schemas');
|
|
38
49
|
test('fetchResultSchema requires saved', () => {
|
|
@@ -68,6 +79,25 @@ test('fixture fable.txt exists and is public-domain self-authored', () => {
|
|
|
68
79
|
assert.ok(!fx.includes('Copyright ©'));
|
|
69
80
|
});
|
|
70
81
|
|
|
82
|
+
// ---------- Test: security scanner (negative/positive samples) ----------
|
|
83
|
+
const { scanDir } = require('../scripts/security-check');
|
|
84
|
+
test('security scanner flags fake secrets and skips clean/ignored files', () => {
|
|
85
|
+
const t = fs.mkdtempSync(path.join(os.tmpdir(), 'drs-sec-'));
|
|
86
|
+
try {
|
|
87
|
+
// 拼接构造,避免源码中出现完整密钥模式(被 security-check 误报)
|
|
88
|
+
fs.writeFileSync(path.join(t, 'bad.js'), 'const key = "' + 'sk-' + 'abcdefghijklmnopqrstuvwxyz123";');
|
|
89
|
+
fs.writeFileSync(path.join(t, 'good.md'), '# clean\nno secrets here\n');
|
|
90
|
+
fs.mkdirSync(path.join(t, 'node_modules'));
|
|
91
|
+
fs.writeFileSync(path.join(t, 'node_modules', 'dep.js'), 'sk-' + 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA');
|
|
92
|
+
const issues = scanDir(t);
|
|
93
|
+
assert.ok(issues.some(i => i.includes('OpenAI-style key')), 'should flag fake key');
|
|
94
|
+
assert.ok(!issues.some(i => i.includes('good.md')), 'clean file should not be flagged');
|
|
95
|
+
assert.ok(!issues.some(i => i.includes('node_modules')), 'node_modules should be skipped');
|
|
96
|
+
} finally {
|
|
97
|
+
fs.rmSync(t, { recursive: true, force: true });
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
|
|
71
101
|
// ---------- Run cache tests ----------
|
|
72
102
|
console.log('\n=== cache tests ===');
|
|
73
103
|
const cacheResult = require('./cache.test.js');
|