deep-read-summarize 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +13 -0
- package/CHANGELOG.md +86 -0
- package/LICENSE +21 -0
- package/README.md +222 -0
- package/SECURITY.md +27 -0
- package/cordis.patch.yml +22 -0
- package/docs/RELEASE-v0.1.0.md +61 -0
- package/docs/RELEASE.md +43 -0
- package/index.js +32 -0
- package/lib/cache.js +62 -0
- package/package.json +66 -0
- package/parsers/book.js +33 -0
- package/parsers/index.js +78 -0
- package/parsers/paper.js +31 -0
- package/parsers/video.js +32 -0
- package/parsers/web.js +30 -0
- package/schemas/index.js +110 -0
- package/scripts/lint.js +23 -0
- package/scripts/security-check.js +32 -0
- package/skills/deep-read-summarize/SKILL.md +68 -0
- package/tests/cache.test.js +69 -0
- package/tests/fixtures/fable.txt +28 -0
- package/tests/index.test.js +80 -0
- package/tests/quick_validate.js +52 -0
- package/tests/run-tests.js +79 -0
- package/workflow.js +43 -0
package/parsers/book.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Parser plugin: book (PDF/EPUB/MOBI text extraction & chunking)
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 独立解析器插件:处理书籍输入。fork 后替换此文件即可支持自定义格式。
|
|
5
|
+
// 接口约定:module.exports = { name, types, buildPrompt(input, opts) }
|
|
6
|
+
// ============================================================================
|
|
7
|
+
|
|
8
|
+
module.exports = {
|
|
9
|
+
name: "book",
|
|
10
|
+
types: ["book"],
|
|
11
|
+
description: "书籍/长篇著作解析器(PDF/EPUB/MOBI → 文本 → 逻辑分块)",
|
|
12
|
+
|
|
13
|
+
// 构建「获取+分块」提示词
|
|
14
|
+
buildPrompt: function (input, opts) {
|
|
15
|
+
const tempDir = opts.tempDir || "./.tmp";
|
|
16
|
+
const tempFile = tempDir + "\\input.txt";
|
|
17
|
+
const maxChunks = opts.maxChunks || 6;
|
|
18
|
+
return [
|
|
19
|
+
"你的任务:获取书籍全文并保存为纯文本,然后给出分块计划。",
|
|
20
|
+
"内容类型:书籍(book)",
|
|
21
|
+
"输入来源:" + input,
|
|
22
|
+
"目标文件:" + tempFile + "(先创建目录 " + tempDir + ",用 write 工具写入)",
|
|
23
|
+
"步骤:",
|
|
24
|
+
"1. 获取全文:PDF→用 read 或 pwsh 调 pdftotext(若为扫描版报告需 OCR);EPUB/MOBI→解包或转文本;纯文本→直接读取。",
|
|
25
|
+
"2. 全文写入目标文件(不截断、不加评注),用 read 确认总行数。",
|
|
26
|
+
"3. 识别书籍结构:目录/序言/各章节标题及其行号范围。",
|
|
27
|
+
"4. 分块:按章节(或章节内小节)切分,每块约 100-200 行或 4000-6000 字,块数不超过 " + maxChunks + " 块;每块给 id/topic/startLine/endLine。",
|
|
28
|
+
"输出 JSON:",
|
|
29
|
+
'{ "kind": "book", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "章节主题", "startLine": 1, "endLine": 200 } ] }',
|
|
30
|
+
"若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
|
|
31
|
+
].join("\n");
|
|
32
|
+
}
|
|
33
|
+
};
|
package/parsers/index.js
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Parser registry — 解析器插件注册表
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 解析器按类型分发,每个输入类型一个文件:
|
|
5
|
+
// - 每个解析器是独立文件(parsers/*.js),可按需增删替换
|
|
6
|
+
// - 本文件负责「按类型发现 + 分发」,workflow 脚本只依赖本接口
|
|
7
|
+
// ============================================================================
|
|
8
|
+
// 用法:
|
|
9
|
+
// const parsers = require("./parsers");
|
|
10
|
+
// const p = parsers.resolve("paper"); // 按类型取解析器
|
|
11
|
+
// const prompt = p.buildPrompt(input, opts);
|
|
12
|
+
// const schema = p.schema; // 结构化输出约束(可选)
|
|
13
|
+
// ============================================================================
|
|
14
|
+
|
|
15
|
+
const fs = require("fs");
|
|
16
|
+
const path = require("path");
|
|
17
|
+
|
|
18
|
+
// 内置解析器(可被用户自定义解析器覆盖)
|
|
19
|
+
const builtins = {
|
|
20
|
+
book: require("./book"),
|
|
21
|
+
paper: require("./paper"),
|
|
22
|
+
video: require("./video"),
|
|
23
|
+
web: require("./web")
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* 解析器注册表
|
|
28
|
+
* 支持在项目根目录创建 custom-parsers/ 目录放置自定义解析器,
|
|
29
|
+
* 同名类型会覆盖内置解析器(用户按需替换)。
|
|
30
|
+
*/
|
|
31
|
+
class ParserRegistry {
|
|
32
|
+
constructor() {
|
|
33
|
+
this.parsers = { ...builtins };
|
|
34
|
+
this._loadCustom();
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
_loadCustom() {
|
|
38
|
+
const customDir = path.join(__dirname, "..", "custom-parsers");
|
|
39
|
+
if (!fs.existsSync(customDir)) return;
|
|
40
|
+
for (const file of fs.readdirSync(customDir)) {
|
|
41
|
+
if (!file.endsWith(".js")) continue;
|
|
42
|
+
try {
|
|
43
|
+
const mod = require(path.join(customDir, file));
|
|
44
|
+
if (mod && mod.name && mod.buildPrompt) {
|
|
45
|
+
// 支持自定义类型名,或覆盖内置类型
|
|
46
|
+
for (const t of mod.types || [mod.name]) {
|
|
47
|
+
this.parsers[t] = mod;
|
|
48
|
+
}
|
|
49
|
+
console.log("[parsers] loaded custom parser:", mod.name);
|
|
50
|
+
}
|
|
51
|
+
} catch (e) {
|
|
52
|
+
console.error("[parsers] failed to load", file, e.message);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** 按类型解析解析器;未知类型回退到 web */
|
|
58
|
+
resolve(type) {
|
|
59
|
+
return this.parsers[type] || this.parsers.web;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** 列出所有已注册解析器 */
|
|
63
|
+
list() {
|
|
64
|
+
return Object.values(this.parsers).map((p) => ({
|
|
65
|
+
name: p.name,
|
|
66
|
+
types: p.types,
|
|
67
|
+
description: p.description
|
|
68
|
+
}));
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const registry = new ParserRegistry();
|
|
73
|
+
|
|
74
|
+
module.exports = {
|
|
75
|
+
resolve: (type) => registry.resolve(type),
|
|
76
|
+
list: () => registry.list(),
|
|
77
|
+
registry
|
|
78
|
+
};
|
package/parsers/paper.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Parser plugin: paper (arXiv/PDF/HTML academic paper)
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 独立解析器插件:处理学术论文输入。fork 后替换此文件即可支持自定义来源。
|
|
5
|
+
// ============================================================================
|
|
6
|
+
|
|
7
|
+
module.exports = {
|
|
8
|
+
name: "paper",
|
|
9
|
+
types: ["paper"],
|
|
10
|
+
description: "学术论文解析器(arXiv/期刊/PDF/HTML)",
|
|
11
|
+
|
|
12
|
+
buildPrompt: function (input, opts) {
|
|
13
|
+
const tempDir = opts.tempDir || "./.tmp";
|
|
14
|
+
const tempFile = tempDir + "\\input.txt";
|
|
15
|
+
const maxChunks = opts.maxChunks || 6;
|
|
16
|
+
return [
|
|
17
|
+
"你的任务:获取论文全文并保存为纯文本,然后给出分块计划。",
|
|
18
|
+
"内容类型:学术论文(paper)",
|
|
19
|
+
"输入来源:" + input,
|
|
20
|
+
"目标文件:" + tempFile + "(先创建目录 " + tempDir + ",用 write 工具写入)",
|
|
21
|
+
"步骤:",
|
|
22
|
+
"1. 获取全文:arXiv→访问 abs 页取 PDF/HTML 全文;PDF→read 或 pdftotext;HTML→提取正文(去导航/广告)。",
|
|
23
|
+
"2. 全文写入目标文件(不截断、不加评注),用 read 确认总行数。",
|
|
24
|
+
"3. 识别论文结构:Abstract/Introduction/Methods/Results/Discussion/Conclusion/Appendix 及各表图行号范围。",
|
|
25
|
+
"4. 分块:按论文章节切分,每块约 100-200 行或 4000-6000 字,块数不超过 " + maxChunks + " 块;每块给 id/topic/startLine/endLine。",
|
|
26
|
+
"输出 JSON:",
|
|
27
|
+
'{ "kind": "paper", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "Introduction", "startLine": 1, "endLine": 200 } ] }',
|
|
28
|
+
"若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
|
|
29
|
+
].join("\n");
|
|
30
|
+
}
|
|
31
|
+
};
|
package/parsers/video.js
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Parser plugin: video (YouTube/Bilibili subtitles & transcripts)
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 独立解析器插件:处理视频输入。优先 yt-dlp 自动抓字幕,失败降级人工转写。
|
|
5
|
+
// ============================================================================
|
|
6
|
+
|
|
7
|
+
module.exports = {
|
|
8
|
+
name: "video",
|
|
9
|
+
types: ["video"],
|
|
10
|
+
description: "视频解析器(YouTube/B站/播客 → 字幕 → 文本)",
|
|
11
|
+
|
|
12
|
+
buildPrompt: function (input, opts) {
|
|
13
|
+
const tempDir = opts.tempDir || "./.tmp";
|
|
14
|
+
const tempFile = tempDir + "\\input.txt";
|
|
15
|
+
const maxChunks = opts.maxChunks || 6;
|
|
16
|
+
return [
|
|
17
|
+
"你的任务:获取视频字幕/转写并保存为纯文本,然后给出分块计划。",
|
|
18
|
+
"内容类型:视频(video)",
|
|
19
|
+
"输入来源:" + input,
|
|
20
|
+
"目标文件:" + tempFile + "(先创建目录 " + tempDir + ",用 write 工具写入)",
|
|
21
|
+
"步骤:",
|
|
22
|
+
"1. 用 pwsh 运行 yt-dlp 抓字幕,命令示例:",
|
|
23
|
+
" yt-dlp --skip-download --write-auto-sub --write-subs --sub-langs 'zh,en' --sub-format vtt/srt -o '" + tempDir + "/sub.%(ext)s' <URL>",
|
|
24
|
+
"2. 读取字幕文件(vtt/srt),去除时间戳与 HTML 标签,转为纯文本写入目标文件。",
|
|
25
|
+
"3. 若 yt-dlp 不可用或抓取失败:输出 { \"saved\": false, \"message\": \"需要人工提供转写文本\" }。",
|
|
26
|
+
"4. 分块:按主题脉络(而非时间)切分,每块约 100-200 行或 4000-6000 字,块数不超过 " + maxChunks + " 块;每块给 id/topic/startLine/endLine。",
|
|
27
|
+
"输出 JSON:",
|
|
28
|
+
'{ "kind": "video", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "主题段", "startLine": 1, "endLine": 200 } ] }',
|
|
29
|
+
"若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
|
|
30
|
+
].join("\n");
|
|
31
|
+
}
|
|
32
|
+
};
|
package/parsers/web.js
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Parser plugin: web (generic web page / article)
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 独立解析器插件:处理网页/文章输入(通用 fallback)。
|
|
5
|
+
// ============================================================================
|
|
6
|
+
|
|
7
|
+
module.exports = {
|
|
8
|
+
name: "web",
|
|
9
|
+
types: ["web"],
|
|
10
|
+
description: "网页/文章解析器(正文提取 → 分块)",
|
|
11
|
+
|
|
12
|
+
buildPrompt: function (input, opts) {
|
|
13
|
+
const tempDir = opts.tempDir || "./.tmp";
|
|
14
|
+
const tempFile = tempDir + "\\input.txt";
|
|
15
|
+
const maxChunks = opts.maxChunks || 6;
|
|
16
|
+
return [
|
|
17
|
+
"你的任务:获取网页/文章正文并保存为纯文本,然后给出分块计划。",
|
|
18
|
+
"内容类型:网页/文章(web)",
|
|
19
|
+
"输入来源:" + input,
|
|
20
|
+
"目标文件:" + tempFile + "(先创建目录 " + tempDir + ",用 write 工具写入)",
|
|
21
|
+
"步骤:",
|
|
22
|
+
"1. 访问链接获取正文文本(去除导航、广告、页脚等噪音),或直接读取本地文本文件。",
|
|
23
|
+
"2. 正文写入目标文件(不截断、不加评注),用 read 确认总行数。",
|
|
24
|
+
"3. 分块:按逻辑主题切分,每块约 100-200 行或 4000-6000 字,块数不超过 " + maxChunks + " 块;每块给 id/topic/startLine/endLine。",
|
|
25
|
+
"输出 JSON:",
|
|
26
|
+
'{ "kind": "web", "saved": true, "totalLines": N, "metadata": { "title": "", "author": "", "year": "", "language": "zh|en" }, "chunkPlan": [ { "id": 1, "topic": "主题", "startLine": 1, "endLine": 200 } ] }',
|
|
27
|
+
"若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
|
|
28
|
+
].join("\n");
|
|
29
|
+
}
|
|
30
|
+
};
|
package/schemas/index.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// JSON Schemas — 结构化输出约束
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 所有子任务产出(获取分块 / 精读 / 成稿元数据)都用 JSON Schema 校验,
|
|
5
|
+
// 不合格自动重试,保证下游(Markdown/Notion/数据库导出)可稳定消费。
|
|
6
|
+
//
|
|
7
|
+
// 注意:本 schema 子集遵循 DSH workflow agent() 的约束
|
|
8
|
+
// (type/properties/required/additionalProperties/items/enum/const/oneOf)。
|
|
9
|
+
// ============================================================================
|
|
10
|
+
|
|
11
|
+
/** 波次1:获取+分块 的结构化输出 */
|
|
12
|
+
const fetchResultSchema = {
|
|
13
|
+
type: "object",
|
|
14
|
+
properties: {
|
|
15
|
+
kind: { type: "string", enum: ["book", "paper", "video", "web"] },
|
|
16
|
+
saved: { type: "boolean" },
|
|
17
|
+
message: { type: "string" },
|
|
18
|
+
totalLines: { type: "number" },
|
|
19
|
+
metadata: {
|
|
20
|
+
type: "object",
|
|
21
|
+
properties: {
|
|
22
|
+
title: { type: "string" },
|
|
23
|
+
author: { type: "string" },
|
|
24
|
+
year: { type: "string" },
|
|
25
|
+
language: { type: "string", enum: ["zh", "en", "other", ""] }
|
|
26
|
+
},
|
|
27
|
+
required: [],
|
|
28
|
+
additionalProperties: true
|
|
29
|
+
},
|
|
30
|
+
chunkPlan: {
|
|
31
|
+
type: "array",
|
|
32
|
+
items: {
|
|
33
|
+
type: "object",
|
|
34
|
+
properties: {
|
|
35
|
+
id: { type: "number" },
|
|
36
|
+
topic: { type: "string" },
|
|
37
|
+
startLine: { type: "number" },
|
|
38
|
+
endLine: { type: "number" }
|
|
39
|
+
},
|
|
40
|
+
required: ["id", "topic", "startLine", "endLine"],
|
|
41
|
+
additionalProperties: false
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
required: ["saved"],
|
|
46
|
+
additionalProperties: false
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/** 波次2:单块精读的结构化输出(可选启用,默认自由文本 + 关键字段) */
|
|
50
|
+
const chunkReadSchema = {
|
|
51
|
+
type: "object",
|
|
52
|
+
properties: {
|
|
53
|
+
chunkId: { type: "number" },
|
|
54
|
+
topic: { type: "string" },
|
|
55
|
+
keyPoints: { type: "array", items: { type: "string" } },
|
|
56
|
+
concepts: {
|
|
57
|
+
type: "array",
|
|
58
|
+
items: {
|
|
59
|
+
type: "object",
|
|
60
|
+
properties: {
|
|
61
|
+
term: { type: "string" },
|
|
62
|
+
definition: { type: "string" }
|
|
63
|
+
},
|
|
64
|
+
required: ["term", "definition"],
|
|
65
|
+
additionalProperties: false
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
citations: {
|
|
69
|
+
type: "array",
|
|
70
|
+
items: {
|
|
71
|
+
type: "object",
|
|
72
|
+
properties: {
|
|
73
|
+
quote: { type: "string" },
|
|
74
|
+
location: { type: "string" },
|
|
75
|
+
page: { type: "string" },
|
|
76
|
+
chapter: { type: "string" },
|
|
77
|
+
timestamp: { type: "string" }
|
|
78
|
+
},
|
|
79
|
+
required: ["quote", "location"],
|
|
80
|
+
additionalProperties: false
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
critique: { type: "string" },
|
|
84
|
+
note: { type: "string" }
|
|
85
|
+
},
|
|
86
|
+
required: ["chunkId", "keyPoints"],
|
|
87
|
+
additionalProperties: false
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
/** 成稿质量自检清单(波次3 内嵌) */
|
|
91
|
+
const qualityChecklistSchema = {
|
|
92
|
+
type: "object",
|
|
93
|
+
properties: {
|
|
94
|
+
coverage: { type: "boolean" },
|
|
95
|
+
citationsReal: { type: "boolean" },
|
|
96
|
+
terminologyConsistent: { type: "boolean" },
|
|
97
|
+
formatComplete: { type: "boolean" },
|
|
98
|
+
lengthMet: { type: "boolean" },
|
|
99
|
+
languageConsistent: { type: "boolean" },
|
|
100
|
+
issues: { type: "array", items: { type: "string" } }
|
|
101
|
+
},
|
|
102
|
+
required: ["coverage", "citationsReal", "terminologyConsistent", "formatComplete", "lengthMet"],
|
|
103
|
+
additionalProperties: false
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
module.exports = {
|
|
107
|
+
fetchResultSchema,
|
|
108
|
+
chunkReadSchema,
|
|
109
|
+
qualityChecklistSchema
|
|
110
|
+
};
|
package/scripts/lint.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// scripts/lint.js — syntax-check all JS files (node --check)
|
|
2
|
+
const { execSync } = require('child_process');
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
function findJs(dir, acc = []) {
|
|
7
|
+
for (const f of fs.readdirSync(dir)) {
|
|
8
|
+
const p = path.join(dir, f);
|
|
9
|
+
const st = fs.statSync(p);
|
|
10
|
+
if (st.isDirectory()) { if (f !== 'node_modules') findJs(p, acc); }
|
|
11
|
+
else if (f.endsWith('.js')) acc.push(p);
|
|
12
|
+
}
|
|
13
|
+
return acc;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const files = findJs(__dirname + '/..');
|
|
17
|
+
let failed = 0;
|
|
18
|
+
for (const f of files) {
|
|
19
|
+
try { execSync('node --check "' + f + '"', { stdio: 'ignore' }); }
|
|
20
|
+
catch (e) { console.error('LINT FAIL:', f); failed++; }
|
|
21
|
+
}
|
|
22
|
+
if (failed) { console.error(failed + ' file(s) failed'); process.exit(1); }
|
|
23
|
+
console.log('Lint OK:', files.length, 'files');
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
|
|
4
|
+
function walk(dir, acc = []) {
|
|
5
|
+
for (const f of fs.readdirSync(dir)) {
|
|
6
|
+
const p = path.join(dir, f);
|
|
7
|
+
const st = fs.statSync(p);
|
|
8
|
+
if (st.isDirectory()) { if (f !== 'node_modules' && f !== '.git') walk(p, acc); }
|
|
9
|
+
else if (/\.(js|md|yml|json|txt)$/.test(f)) acc.push(p);
|
|
10
|
+
}
|
|
11
|
+
return acc;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const files = walk(process.cwd());
|
|
15
|
+
const issues = [];
|
|
16
|
+
const patterns = [
|
|
17
|
+
[/api[_-]?key\s*[:=]\s*['"][^'"]+/i, 'API KEY pattern'],
|
|
18
|
+
[/sk-[A-Za-z0-9]{20,}/, 'OpenAI-style key'],
|
|
19
|
+
[/password\s*[:=]\s*['"][^'"]+/i, 'password pattern'],
|
|
20
|
+
[/C:\\Users\\[^\\"']+/i, 'Windows user path'],
|
|
21
|
+
[/AKIA[0-9A-Z]{16}/, 'AWS key']
|
|
22
|
+
];
|
|
23
|
+
|
|
24
|
+
for (const f of files) {
|
|
25
|
+
const c = fs.readFileSync(f, 'utf8');
|
|
26
|
+
for (const [re, label] of patterns) {
|
|
27
|
+
if (re.test(c)) issues.push(label + ' in ' + path.relative(process.cwd(), f));
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (issues.length === 0) { console.log('OK: no secrets/passwords/local user paths found'); }
|
|
32
|
+
else { issues.forEach(i => console.log('ISSUE: ' + i)); process.exit(1); }
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: deep-read-summarize
|
|
3
|
+
description: 深度精读并总结书籍/学术论文/视频/网页,输出带 YAML frontmatter 的 Obsidian 笔记(2500+ 字,含术语表/批判分析/行动要点/延伸阅读)
|
|
4
|
+
whenToUse: 用户提供一本书、论文(arXiv/PDF)、视频链接(YouTube/B站)或网页,要求深度精读、提取要点、生成可存入 Obsidian 的精读笔记时
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# deep-read-summarize — 深度精读与总结
|
|
8
|
+
|
|
9
|
+
把一本书、一篇论文、一个视频或网页,精读成一篇结构化的 Obsidian 笔记。
|
|
10
|
+
|
|
11
|
+
## 能力
|
|
12
|
+
|
|
13
|
+
- 书籍(PDF/EPUB/MOBI)、论文(arXiv/PDF/HTML)、视频(yt-dlp 字幕)、网页
|
|
14
|
+
- 长内容分块后由并行子代理精读(MapReduce),不溢出上下文
|
|
15
|
+
- JSON Schema 约束子任务输出,不合格自动重试
|
|
16
|
+
- 关键引用标注页码/章节/段落,降低编造风险
|
|
17
|
+
- 配置错误终止(FATAL);内容解析失败降级标记缺口继续
|
|
18
|
+
- 输出带 YAML frontmatter(type 字段可配 Dataview)
|
|
19
|
+
|
|
20
|
+
## 输入
|
|
21
|
+
|
|
22
|
+
用户提供:内容链接或本地文件路径。
|
|
23
|
+
可选参数:`type`(auto/book/paper/video/web)、`options`(minWords/fastMode/maxChunks/requireCitations/includeTimestamps/outputDir)。
|
|
24
|
+
|
|
25
|
+
## 执行步骤
|
|
26
|
+
|
|
27
|
+
1. **配置校验**:缺 input 或 type 非法 → 报 FATAL 终止
|
|
28
|
+
2. **幂等检查**:若输入已处理过(args._processedKeys 命中)→ 直接返回缓存结果
|
|
29
|
+
3. **解析器选择**:按类型从 parsers 注册表选解析器(book/paper/video/web)
|
|
30
|
+
4. **波次1 获取+分块**:解析器获取全文 → 写入临时文件 → 生成分块计划(JSON Schema 校验)
|
|
31
|
+
5. **波次2 并行精读**:每个分块一个子代理深度精读(Map)
|
|
32
|
+
6. **波次3 合并成稿**:整合为完整笔记(Reduce),内嵌质量自检
|
|
33
|
+
7. **质量校验**:可选重试(maxRetries),检查覆盖度/引用真实性/术语一致/格式完整/篇幅/语言
|
|
34
|
+
|
|
35
|
+
## 输出
|
|
36
|
+
|
|
37
|
+
Markdown 笔记(默认写到 `./output/`,可用 `options.outputDir` 指向 Obsidian 仓库)
|
|
38
|
+
|
|
39
|
+
```markdown
|
|
40
|
+
---
|
|
41
|
+
title: "《XXX》深度精读笔记"
|
|
42
|
+
author: 作者
|
|
43
|
+
year: 年份
|
|
44
|
+
type: book | paper | video | web
|
|
45
|
+
url: 来源
|
|
46
|
+
tags: [精读, ...]
|
|
47
|
+
created: 日期
|
|
48
|
+
status: 已完成
|
|
49
|
+
---
|
|
50
|
+
# 《XXX》深度精读笔记
|
|
51
|
+
## 1. 概述
|
|
52
|
+
## 2. 结构拆解
|
|
53
|
+
## 3. 关键概念与术语表
|
|
54
|
+
## 4. 关键引用
|
|
55
|
+
## 5. 批判性分析
|
|
56
|
+
## 6. 个人思考与应用
|
|
57
|
+
## 7. 延伸阅读
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## 视频说明
|
|
61
|
+
|
|
62
|
+
- 优先 yt-dlp 抓字幕(`winget install yt-dlp.yt-dlp`)
|
|
63
|
+
- 抓不到时提示用户手动提供转写文本
|
|
64
|
+
- 默认不标时间戳(`options.includeTimestamps` 可开)
|
|
65
|
+
|
|
66
|
+
## 参考
|
|
67
|
+
|
|
68
|
+
项目主页:https://github.com/PensiveFei/deep-read-summarize
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// tests/cache.test.js — idempotency cache tests
|
|
2
|
+
const assert = require('assert');
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const os = require('os');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const cache = require('../lib/cache');
|
|
7
|
+
|
|
8
|
+
let passed = 0, failed = 0;
|
|
9
|
+
function test(name, fn) {
|
|
10
|
+
try { fn(); passed++; console.log(' ✅', name); }
|
|
11
|
+
catch (e) { failed++; console.error(' ❌', name, '—', e.message); }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const tmp = path.join(os.tmpdir(), 'drs-cache-test-' + Date.now());
|
|
15
|
+
const cacheFile = path.join(tmp, 'cache.jsonl');
|
|
16
|
+
|
|
17
|
+
test('fingerprint is deterministic for same input', () => {
|
|
18
|
+
const a = cache.fingerprint('https://arxiv.org/abs/2307.09042');
|
|
19
|
+
const b = cache.fingerprint('https://arxiv.org/abs/2307.09042');
|
|
20
|
+
assert.strictEqual(a, b);
|
|
21
|
+
assert.ok(a && a.length >= 16);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test('fingerprint differs for different inputs', () => {
|
|
25
|
+
const a = cache.fingerprint('input-a');
|
|
26
|
+
const b = cache.fingerprint('input-b');
|
|
27
|
+
assert.notStrictEqual(a, b);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test('fingerprint includes file meta for local files', () => {
|
|
31
|
+
const f = path.join(tmp, 'sample.txt');
|
|
32
|
+
fs.mkdirSync(tmp, { recursive: true });
|
|
33
|
+
fs.writeFileSync(f, 'hello');
|
|
34
|
+
const st = fs.statSync(f);
|
|
35
|
+
const a = cache.fingerprint(f, { size: st.size, mtimeMs: st.mtimeMs });
|
|
36
|
+
const b = cache.fingerprint(f, { size: st.size + 1, mtimeMs: st.mtimeMs });
|
|
37
|
+
assert.notStrictEqual(a, b, 'size change should change fingerprint');
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test('hasProcessed returns false for unknown key', () => {
|
|
41
|
+
assert.strictEqual(cache.hasProcessed(cacheFile, 'unknown-key'), false);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test('markProcessed then hasProcessed returns true', () => {
|
|
45
|
+
const key = cache.fingerprint('https://example.com/doc');
|
|
46
|
+
cache.markProcessed(cacheFile, key, '/tmp/out.md');
|
|
47
|
+
assert.strictEqual(cache.hasProcessed(cacheFile, key), true);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test('cache file is JSONL (append-only)', () => {
|
|
51
|
+
const raw = fs.readFileSync(cacheFile, 'utf8').trim().split('\n');
|
|
52
|
+
assert.ok(raw.length >= 1);
|
|
53
|
+
const rec = JSON.parse(raw[0]);
|
|
54
|
+
assert.ok(rec.key && rec.outputPath && rec.ts);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
test('corrupted line is skipped', () => {
|
|
58
|
+
fs.appendFileSync(cacheFile, '{broken json}\n', 'utf8');
|
|
59
|
+
const map = cache.readCache(cacheFile);
|
|
60
|
+
assert.ok(map.size >= 1, 'should still read valid lines');
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
// cleanup
|
|
64
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
65
|
+
|
|
66
|
+
console.log('');
|
|
67
|
+
console.log(passed + ' cache tests passed, ' + failed + ' failed');
|
|
68
|
+
if (require.main === module) process.exit(failed ? 1 : 0);
|
|
69
|
+
module.exports = { passed, failed };
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# The Little Fox and the Stars
|
|
2
|
+
|
|
3
|
+
A public-domain style fable by the deep-read-summarize team (CC0 / self-authored for testing).
|
|
4
|
+
|
|
5
|
+
## Chapter 1: The Forest
|
|
6
|
+
|
|
7
|
+
Once there was a little fox named Pip who lived in a forest where the stars came down to drink from the river at midnight. Every night, Pip watched them shimmer and dance, and he wished he could join them.
|
|
8
|
+
|
|
9
|
+
One evening, a star with a crooked tail stumbled onto the riverbank, too tired to fly back up. "Please," said the star, "help me reach the sky again."
|
|
10
|
+
|
|
11
|
+
Pip thought for a moment. The star was heavy and the sky was far. But Pip remembered that the tallest tree in the forest touched the clouds on windy nights.
|
|
12
|
+
|
|
13
|
+
## Chapter 2: The Climb
|
|
14
|
+
|
|
15
|
+
Pip carried the star to the Great Oak and began to climb. The branches were slippery with dew, and twice the star nearly fell. "Hold tight," said Pip, "we are almost there."
|
|
16
|
+
|
|
17
|
+
At the top, the wind blew strong and the sky opened wide. Pip lifted the star with all his strength, and the star floated up, glowing brighter with each heartbeat.
|
|
18
|
+
|
|
19
|
+
## Chapter 3: The Promise
|
|
20
|
+
|
|
21
|
+
"Come back tomorrow night," called the star, "and I will teach you how to shine."
|
|
22
|
+
|
|
23
|
+
And Pip did. He learned that even the smallest fox can carry a little light, as long as he climbs one branch at a time.
|
|
24
|
+
|
|
25
|
+
THE END
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
This fable was written solely for software testing and is released into the public domain (CC0).
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// tests/index.test.js — node:test 风格测试(dsh.so / npm 规范)
|
|
2
|
+
// 运行: node --test
|
|
3
|
+
const { test } = require('node:test');
|
|
4
|
+
const assert = require('node:assert');
|
|
5
|
+
const fs = require('fs');
|
|
6
|
+
const path = require('path');
|
|
7
|
+
|
|
8
|
+
// ---------- 插件入口 ----------
|
|
9
|
+
test('plugin loads with name deep-read-summarize', () => {
|
|
10
|
+
const p = require('../index.js');
|
|
11
|
+
const def = p.load();
|
|
12
|
+
assert.strictEqual(def.name, 'deep-read-summarize');
|
|
13
|
+
assert.strictEqual(def.workflow.meta.name, 'deep-read-summarize');
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
// ---------- 解析器 ----------
|
|
17
|
+
const parsers = require('../parsers');
|
|
18
|
+
test('registry exposes 4 built-in parsers', () => {
|
|
19
|
+
const names = parsers.list().map(p => p.name).sort();
|
|
20
|
+
assert.deepStrictEqual(names, ['book', 'paper', 'video', 'web']);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test('resolve falls back to web for unknown type', () => {
|
|
24
|
+
assert.strictEqual(parsers.resolve('unknown').name, 'web');
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test('every parser builds a non-empty prompt', () => {
|
|
28
|
+
for (const type of ['book', 'paper', 'video', 'web']) {
|
|
29
|
+
const prompt = parsers.resolve(type).buildPrompt('https://example.com/x', { tempDir: '/tmp', tempFile: '/tmp/in.txt', maxChunks: 4 });
|
|
30
|
+
assert.ok(typeof prompt === 'string' && prompt.length > 50, type + ' prompt too short');
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
// ---------- schemas ----------
|
|
35
|
+
const schemas = require('../schemas');
|
|
36
|
+
test('fetchResultSchema requires saved', () => {
|
|
37
|
+
assert.ok(schemas.fetchResultSchema.required.includes('saved'));
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
// ---------- workflow ----------
|
|
41
|
+
const wf = require('../workflow');
|
|
42
|
+
test('workflow meta is valid', () => {
|
|
43
|
+
assert.strictEqual(wf.meta.name, 'deep-read-summarize');
|
|
44
|
+
assert.ok(wf.meta.phases.length >= 3);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test('workflow script parses as async body', () => {
|
|
48
|
+
const fn = new Function('args','phase','agent','parallel','log', 'return (async () => { ' + wf.script + ' })()');
|
|
49
|
+
assert.ok(typeof fn === 'function');
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// ---------- 幂等缓存 ----------
|
|
53
|
+
const cache = require('../lib/cache');
|
|
54
|
+
const os = require('os');
|
|
55
|
+
test('cache fingerprint is deterministic', () => {
|
|
56
|
+
assert.strictEqual(
|
|
57
|
+
cache.fingerprint('https://arxiv.org/abs/2307.09042'),
|
|
58
|
+
cache.fingerprint('https://arxiv.org/abs/2307.09042')
|
|
59
|
+
);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
// ---------- 打包完整性 ----------
|
|
63
|
+
test('cordis.patch.yml exists and uses insert block', () => {
|
|
64
|
+
const patch = fs.readFileSync(path.join(__dirname, '..', 'cordis.patch.yml'), 'utf8');
|
|
65
|
+
assert.ok(patch.includes('- insert:'));
|
|
66
|
+
assert.ok(patch.includes('deep-read-summarize'));
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test('skills bundle exists', () => {
|
|
70
|
+
const skill = path.join(__dirname, '..', 'skills', 'deep-read-summarize', 'SKILL.md');
|
|
71
|
+
assert.ok(fs.existsSync(skill), 'SKILL.md missing');
|
|
72
|
+
const content = fs.readFileSync(skill, 'utf8');
|
|
73
|
+
assert.ok(content.includes('name: deep-read-summarize'));
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// ---------- fixture 版权 ----------
|
|
77
|
+
test('fixture is self-authored public-domain', () => {
|
|
78
|
+
const fx = fs.readFileSync(path.join(__dirname, 'fixtures', 'fable.txt'), 'utf8');
|
|
79
|
+
assert.ok(fx.includes('released into the public domain'));
|
|
80
|
+
});
|