deep-read-summarize 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +13 -0
- package/CHANGELOG.md +86 -0
- package/LICENSE +21 -0
- package/README.md +222 -0
- package/SECURITY.md +27 -0
- package/cordis.patch.yml +22 -0
- package/docs/RELEASE-v0.1.0.md +61 -0
- package/docs/RELEASE.md +43 -0
- package/index.js +32 -0
- package/lib/cache.js +62 -0
- package/package.json +66 -0
- package/parsers/book.js +33 -0
- package/parsers/index.js +78 -0
- package/parsers/paper.js +31 -0
- package/parsers/video.js +32 -0
- package/parsers/web.js +30 -0
- package/schemas/index.js +110 -0
- package/scripts/lint.js +23 -0
- package/scripts/security-check.js +32 -0
- package/skills/deep-read-summarize/SKILL.md +68 -0
- package/tests/cache.test.js +69 -0
- package/tests/fixtures/fable.txt +28 -0
- package/tests/index.test.js +80 -0
- package/tests/quick_validate.js +52 -0
- package/tests/run-tests.js +79 -0
- package/workflow.js +43 -0
package/.env.example
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# .env.example — 复制为 .env 后填写
|
|
2
|
+
# 本项目零依赖,不需要 API key 也能运行核心流程。
|
|
3
|
+
# 以下为可选配置(如你的 DSH 环境需要自定义)
|
|
4
|
+
|
|
5
|
+
# 可选:DeepSeek/LLM API 配置(占位符,仅示例)
|
|
6
|
+
DEEPSEEK_API_KEY=your-api-key-here
|
|
7
|
+
DEEPSEEK_BASE_URL=https://api.example.com/v1
|
|
8
|
+
|
|
9
|
+
# 可选:输出目录覆盖
|
|
10
|
+
OUTPUT_DIR=./output
|
|
11
|
+
|
|
12
|
+
# 可选:临时目录
|
|
13
|
+
TEMP_DIR=./_deepread_temp
|
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/).
|
|
5
|
+
|
|
6
|
+
## [0.2.0] — 2026-08-21
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
- 首次发布到 npm registry(`npm install deep-read-summarize`),含此前未发布的 0.1.1 变更
|
|
10
|
+
- prepublishOnly 质量门禁:发布前自动跑 17 项测试 + lint + 安全检查
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
- package.json version 对齐 git tag(0.1.0 → 0.2.0)
|
|
14
|
+
- keywords 移除 `deepseek-harness` 商标词(安全合规要求,与 v0.1.0 决策一致)
|
|
15
|
+
- README 补充 npm 安装方式
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
- git 历史整理:以本地完整开发历史为准 force push 覆盖 API 拼凑历史,补上缺失的 `.github/workflows/ci.yml`
|
|
19
|
+
|
|
20
|
+
## [0.1.1] — 2026-08-19
|
|
21
|
+
|
|
22
|
+
### Added
|
|
23
|
+
- DSH 插件化:cordis.patch.yml(bundle patch)、index.js 插件入口、skills/ 打包
|
|
24
|
+
- node:test 风格测试(tests/index.test.js,`npm run test:node`)
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
- package.json:dsh.bundle 字段、exports、repository、keywords 完善
|
|
28
|
+
- 废弃 cordis.yml → 由 cordis.patch.yml 替代
|
|
29
|
+
|
|
30
|
+
## [0.1.0] — 2026-08-18
|
|
31
|
+
|
|
32
|
+
### Added
|
|
33
|
+
- 首个公开预发布版(pre-release)
|
|
34
|
+
- 四种输入解析器插件:书籍/论文/视频/网页
|
|
35
|
+
- 三波次 MapReduce 精读流水线
|
|
36
|
+
- JSON Schema 结构化输出约束
|
|
37
|
+
- 幂等缓存(lib/cache.js)
|
|
38
|
+
- 失败分级(FATAL vs 可降级)
|
|
39
|
+
- 质量校验循环
|
|
40
|
+
- CI(lint + 测试 + Gitleaks 密钥扫描)
|
|
41
|
+
- README/SECURITY/CONTRIBUTING/CHANGELOG 文档
|
|
42
|
+
|
|
43
|
+
## [3.0.0] — 2026-08-18
|
|
44
|
+
|
|
45
|
+
### Added
|
|
46
|
+
- Plugin parser architecture: `parsers/` registry with book/paper/video/web adapters
|
|
47
|
+
- JSON Schema structured output constraints (`schemas/index.js`)
|
|
48
|
+
- Failure discipline: FATAL (config) vs degradable (content) error classes
|
|
49
|
+
- Quality control loop with retry (`maxRetries`)
|
|
50
|
+
- Configurable citations (`requireCitations`) and video timestamps (`includeTimestamps`)
|
|
51
|
+
- Fixture-driven test suite (`npm test`, 10 tests)
|
|
52
|
+
- Quick validation script (`npm run validate`)
|
|
53
|
+
- CI workflow (lint + tests)
|
|
54
|
+
- cordis.yml plugin tree example
|
|
55
|
+
|
|
56
|
+
### Changed
|
|
57
|
+
- Workflow reduced to 3 serial waves (fetch/chunk → parallel read → merge+QC)
|
|
58
|
+
- Subagent count reduced to N+2~3
|
|
59
|
+
- Defaults changed to general-purpose: `requireCitations: true`, `includeTimestamps: false`
|
|
60
|
+
|
|
61
|
+
### Removed
|
|
62
|
+
- Standalone quality-check subagent (merged into wave 3 + optional lightweight retry)
|
|
63
|
+
|
|
64
|
+
## [2.1.0] — 2026-08-18
|
|
65
|
+
|
|
66
|
+
### Added
|
|
67
|
+
- One-click write-to-Obsidian from the draft agent
|
|
68
|
+
- fastMode (skip sections 5-7)
|
|
69
|
+
|
|
70
|
+
## [2.0.0] — 2026-08-18
|
|
71
|
+
|
|
72
|
+
### Changed
|
|
73
|
+
- Merged identify+fetch+chunk into one subagent (3 waves total)
|
|
74
|
+
- Removed standalone QC subagent
|
|
75
|
+
|
|
76
|
+
## [1.1.0] — 2026-08-18
|
|
77
|
+
|
|
78
|
+
### Fixed
|
|
79
|
+
- Long-content overflow: write-to-temp-file + line-range chunking
|
|
80
|
+
|
|
81
|
+
## [1.0.0] — 2026-08-18
|
|
82
|
+
|
|
83
|
+
### Added
|
|
84
|
+
- Initial deep-reading pipeline (5 serial waves)
|
|
85
|
+
- Template prompts for books/papers/videos/web
|
|
86
|
+
- Obsidian note template with YAML frontmatter
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 deep-read-summarize contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
# deep-read-summarize
|
|
2
|
+
> **Disclaimer**: This is an **unofficial third-party tool**. It is not affiliated with, endorsed by, or sponsored by DeepSeek or the DeepSeek Harness project. "DeepSeek" and "DeepSeek Harness" are trademarks of their respective owners. This project only orchestrates the workflow tool available in your DSH environment; it does not redistribute any DeepSeek software.
|
|
3
|
+
|
|
4
|
+
给 DSH(DeepSeek Harness)写的一个精读工作流:输入一本书、一篇论文、一个视频链接或网页,输出一份结构化的 Obsidian 笔记。
|
|
5
|
+
|
|
6
|
+
内容会被拆成若干块,由多个子代理并行精读,再合并成一篇带 YAML frontmatter 的 Markdown。关键结论附原文出处,成稿前有一道质量校验。
|
|
7
|
+
|
|
8
|
+
> 注意:DSH 目前是 developer preview,接口可能变化。本仓库针对特定版本的 workflow 工具语义编写,见下文[兼容性](#兼容性)。
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 能做什么
|
|
13
|
+
|
|
14
|
+
- 书籍(PDF/EPUB/MOBI)、论文(arXiv/PDF/HTML)、视频(YouTube/B 站字幕)、网页
|
|
15
|
+
- 长内容分块后由并行子代理精读,再合并
|
|
16
|
+
- 各子任务的输出用 JSON Schema 约束,不合格自动重试
|
|
17
|
+
- 关键引用必须标注页码/章节/段落,降低编造风险
|
|
18
|
+
- 配置错误直接报错终止;某块内容解析失败则跳过并标记缺口
|
|
19
|
+
- 输出可直接放进 Obsidian,配合 Dataview 使用
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
## 结构
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
输入(链接或文件路径)
|
|
27
|
+
│
|
|
28
|
+
▼
|
|
29
|
+
解析器注册表 ── book / paper / video / web
|
|
30
|
+
│
|
|
31
|
+
▼
|
|
32
|
+
波次1 获取内容 → 写入临时文件 → 生成分块计划
|
|
33
|
+
│
|
|
34
|
+
▼
|
|
35
|
+
波次2 N 个子代理并行精读各块(Map)
|
|
36
|
+
│
|
|
37
|
+
▼
|
|
38
|
+
波次3 合并成稿 + 质量校验(Reduce)
|
|
39
|
+
│
|
|
40
|
+
▼
|
|
41
|
+
Obsidian 笔记
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
三个波次,子代理总数约 N+2 个,N 是分块数。
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
parsers/ 各输入类型的解析器,按类型分发
|
|
48
|
+
book.js 书籍:PDF/EPUB/MOBI 文本提取、章节分块
|
|
49
|
+
paper.js 论文:arXiv/PDF/HTML 结构识别
|
|
50
|
+
video.js 视频:yt-dlp 抓字幕、转写清洗
|
|
51
|
+
web.js 网页:正文提取
|
|
52
|
+
index.js 注册表:解析器发现与回退
|
|
53
|
+
schemas/ 子任务输出的 JSON Schema
|
|
54
|
+
scripts/ lint、安全检查
|
|
55
|
+
tests/ fixture 测试与验证脚本
|
|
56
|
+
workflow.js workflow 脚本本体(meta + script)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
想换某个输入类型的处理方式,在 `custom-parsers/` 放一个同接口的解析器即可,同名类型会覆盖内置实现。接口只有三个字段:`name`、`types`、`buildPrompt(input, opts)`。
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## dsh.so 生态
|
|
64
|
+
|
|
65
|
+
本仓库带 `dsh-plugin` topic 和 `package.json` 的 `dsh` 字段,可被 dsh.so 注册表索引。
|
|
66
|
+
|
|
67
|
+
dsh.so 收录要求:公开仓库、`dsh-plugin` topic(或 dsh 字段)、README 安装说明、SPDX 许可证(MIT)。
|
|
68
|
+
提交后标记为 **Declared**(自声明兼容);如果你实际使用并验证兼容,请在官方 Discussions 报告结果,这是唯一升 **Verified** 的途径。
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
## 安装(DSH 插件)
|
|
72
|
+
|
|
73
|
+
deep-read-summarize 是 DSH 插件,可通过 npm、dsh.so 生态或本地安装:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
# npm 安装(已发布到 registry,无第三方依赖)
|
|
77
|
+
npm install deep-read-summarize
|
|
78
|
+
|
|
79
|
+
# 或本地安装(dsh profile 目录)
|
|
80
|
+
pnpm add ./deep-read-summarize-0.2.0.tgz
|
|
81
|
+
# 然后在 dsh 配置的 dsh.profile.bundles 追加:
|
|
82
|
+
# - deep-read-summarize
|
|
83
|
+
# 重启 dsh web 即可(POST /dsh-market/restart)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
安装后自动注册:
|
|
87
|
+
- `deep-read-summarize` workflow(meta + script)
|
|
88
|
+
- `deep-read-summarize` 技能(`skills/deep-read-summarize/SKILL.md`)
|
|
89
|
+
- 四种解析器(`parsers/`)与 JSON Schema(`schemas/`)
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
## 快速开始(约 5 分钟)
|
|
93
|
+
|
|
94
|
+
### 1. 安装
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
# 方式 A:npm 直接安装(无需 clone)
|
|
98
|
+
npm install deep-read-summarize
|
|
99
|
+
|
|
100
|
+
# 方式 B:从源码开发
|
|
101
|
+
git clone https://github.com/<your-org>/deep-read-summarize.git
|
|
102
|
+
cd deep-read-summarize
|
|
103
|
+
npm install # 无第三方依赖,仅初始化
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### 2. 验证环境(离线,不需要任何 API key)
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
npm test # 17 项 fixture 测试,全部离线跑通
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
看到 `TOTAL: 17 passed, 0 failed` 即环境就绪。
|
|
113
|
+
|
|
114
|
+
### 3. 一行命令喂入 demo 文件
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
node -e "const wf = require('./workflow.js'); console.log('meta:', wf.meta.name); console.log('parsers:', wf.parsers.list().map(p => p.name).join(', '));"
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
输出类似:
|
|
121
|
+
|
|
122
|
+
```
|
|
123
|
+
meta: deep-read-summarize
|
|
124
|
+
parsers: book, paper, video, web
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### 4. 真实运行(需要 DSH workflow 工具)
|
|
128
|
+
|
|
129
|
+
把下面的 JSON 传给 DSH 的 workflow 工具(见下节「用法」):
|
|
130
|
+
|
|
131
|
+
```jsonc
|
|
132
|
+
{ "input": "https://arxiv.org/abs/2307.09042", "type": "paper", "options": { "maxChunks": 4, "fastMode": true } }
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
工作流会返回结构化结果:`{ ok, kind, title, filePath, qualityPassed, note }`,`note` 即最终 Markdown 笔记。
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
## 用法
|
|
139
|
+
|
|
140
|
+
把下面这段 JSON 传给 DSH 的 workflow 工具:
|
|
141
|
+
|
|
142
|
+
```jsonc
|
|
143
|
+
{
|
|
144
|
+
"input": "https://arxiv.org/abs/2307.09042", // 链接或文件路径
|
|
145
|
+
"type": "auto", // auto | book | paper | video | web
|
|
146
|
+
"options": {
|
|
147
|
+
"minWords": 2500,
|
|
148
|
+
"fastMode": false, // true 时跳过 5-7 节,速度快一些
|
|
149
|
+
"maxChunks": 6, // 分块上限,1-12
|
|
150
|
+
"requireCitations": true, // 关键结论是否必须标注出处
|
|
151
|
+
"includeTimestamps": false,
|
|
152
|
+
"outputDir": "./output", // 笔记输出目录(可指向 Obsidian 仓库)
|
|
153
|
+
"tempDir": "./.tmp" // 临时文件目录
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
示例:
|
|
159
|
+
|
|
160
|
+
| 类型 | input |
|
|
161
|
+
|------|-------|
|
|
162
|
+
| 论文 | `https://arxiv.org/abs/2307.09042` |
|
|
163
|
+
| 书籍 | `D:/books/xxx.pdf` |
|
|
164
|
+
| 视频 | `https://youtube.com/watch?v=xxx` |
|
|
165
|
+
| 网页 | `https://example.com/article` |
|
|
166
|
+
|
|
167
|
+
输出写到 `D:/Obsidian 仓库/PensiveFei/精读笔记/<标题>.md`,路径在 `workflow.js` 里可以改。
|
|
168
|
+
|
|
169
|
+
视频字幕需要本机装 yt-dlp(`winget install yt-dlp.yt-dlp`)。没装也不影响其他类型;抓不到字幕时工作流会提示你手动提供转写文本。
|
|
170
|
+
|
|
171
|
+
---
|
|
172
|
+
|
|
173
|
+
## 失败怎么处理
|
|
174
|
+
|
|
175
|
+
分两类:
|
|
176
|
+
|
|
177
|
+
- **配置错误**(缺 input、type 非法、options 格式错):直接抛异常终止,不产出半成品。
|
|
178
|
+
- **内容问题**(抓取失败、某块精读失败):返回 `{ ok: false, stage, fatal: false }`,由调用方决定。某一块失败时跳过它、在成稿里标注缺口,不整体中断。
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
## 成本
|
|
183
|
+
|
|
184
|
+
一篇论文、6 块、完整模式,大约 15-25k token。用 `fastMode` 并调低 `maxChunks` 能省四成左右。
|
|
185
|
+
|
|
186
|
+
---
|
|
187
|
+
|
|
188
|
+
## 安全与版权
|
|
189
|
+
|
|
190
|
+
- 抓取视频/网页会发起外部请求,由子代理在 DSH 的沙箱和审批策略下执行;不要让模型无人值守地跑任意脚本。
|
|
191
|
+
- 子代理写文件需要相应权限;如果权限不够,workflow 会把内容返回,由主代理负责落盘。
|
|
192
|
+
- 这个仓库只有流程本身:workflow 定义、解析器代码、提示词模板、schema。**不含任何受版权保护内容的提取结果。** 测试用的 fixture 是自写的公共领域寓言。用本工作流处理有版权的材料时,产出物的使用责任在你。
|
|
193
|
+
|
|
194
|
+
---
|
|
195
|
+
|
|
196
|
+
## 开发
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
npm install
|
|
200
|
+
npm run lint # node --check 全部 JS
|
|
201
|
+
npm test # fixture 测试
|
|
202
|
+
npm run validate # 发布前验证(含安全检查)
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
改动说明见 CHANGELOG.md,贡献规范见 CONTRIBUTING.md。
|
|
206
|
+
|
|
207
|
+
---
|
|
208
|
+
|
|
209
|
+
## 兼容性
|
|
210
|
+
|
|
211
|
+
DSH 还在快速迭代,有过破坏性变更。本仓库的依赖面是:
|
|
212
|
+
|
|
213
|
+
- workflow 工具的 `agent()`、`parallel()`、`phase()`、`log()`、`args`
|
|
214
|
+
- JSON Schema 子集:`type / properties / required / additionalProperties / items / enum / const / oneOf`
|
|
215
|
+
|
|
216
|
+
升级 DSH 后先跑一遍 `npm test`。如果坏了,对照 CHANGELOG.md 里的版本记录排查。
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## License
|
|
221
|
+
|
|
222
|
+
MIT,见 [LICENSE](LICENSE)。
|
package/SECURITY.md
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Reporting a Vulnerability
|
|
4
|
+
|
|
5
|
+
Please do **not** open a public issue for security vulnerabilities.
|
|
6
|
+
|
|
7
|
+
To report a vulnerability privately:
|
|
8
|
+
|
|
9
|
+
1. Open a [private security advisory](https://github.com/<your-org>/deep-read-summarize/security/advisories/new) (if available), or
|
|
10
|
+
2. Email the maintainer (see repository profile for contact).
|
|
11
|
+
|
|
12
|
+
## What to include
|
|
13
|
+
|
|
14
|
+
- Affected version / commit
|
|
15
|
+
- Steps to reproduce
|
|
16
|
+
- Impact description
|
|
17
|
+
- Suggested fix (optional)
|
|
18
|
+
|
|
19
|
+
## Response timeline
|
|
20
|
+
|
|
21
|
+
- Acknowledgment within 3 business days
|
|
22
|
+
- Initial assessment within 7 business days
|
|
23
|
+
- Fix and release within 30 days (depending on severity)
|
|
24
|
+
|
|
25
|
+
## Scope
|
|
26
|
+
|
|
27
|
+
This project only ships workflow code, prompt templates, and schemas. It does not contain copyrighted content or secrets. Vulnerabilities of interest include prompt injection in parser prompts, sandbox escape attempts, and unsafe file handling.
|
package/cordis.patch.yml
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# dsh bundle patch: inserts this plugin's loader entry into a profile layer.
|
|
2
|
+
# Bundle patches add rows via `insert` blocks; top-level rows would be treated
|
|
3
|
+
# as id-targeted overrides of EXISTING entries ("entry not found" otherwise).
|
|
4
|
+
#
|
|
5
|
+
# This plugin is a workflow + skill provider: it registers the deep-read-summarize
|
|
6
|
+
# workflow (meta/script) and the filesystem skill, so users can deep-read books,
|
|
7
|
+
# papers, videos and web pages into structured Obsidian notes.
|
|
8
|
+
|
|
9
|
+
- insert:
|
|
10
|
+
- id: deep-read-summarize
|
|
11
|
+
name: 'deep-read-summarize'
|
|
12
|
+
|
|
13
|
+
# All plugin config keys are optional; defaults are applied by the plugin.
|
|
14
|
+
# config:
|
|
15
|
+
# outputDir: './output' # where notes are written (default: ./output)
|
|
16
|
+
# tempDir: './.tmp' # temp files for chunked content (default: ./.tmp)
|
|
17
|
+
# minWords: 2500 # minimum output length
|
|
18
|
+
# fastMode: false # skip sections 5-7 for speed
|
|
19
|
+
# maxChunks: 6 # chunk cap (1-12)
|
|
20
|
+
# requireCitations: true # key conclusions must cite source location
|
|
21
|
+
# includeTimestamps: false # video timestamps (off by default)
|
|
22
|
+
# cache: true # idempotency cache on input fingerprint
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# v0.1.0 — 首个预发布版
|
|
2
|
+
|
|
3
|
+
> **锁定的 DeepSeek Harness 版本**:workflow 工具语义(`agent()` / `parallel()` / `phase()` / `log()` / `args`)。
|
|
4
|
+
> **兼容性**:0.x 为早期迭代,接口可能变化;升级前重跑 `npm test`。
|
|
5
|
+
|
|
6
|
+
## 发布说明
|
|
7
|
+
|
|
8
|
+
deep-read-summarize 是一个给 DSH(DeepSeek Harness)用的精读工作流:输入一本书、一篇论文、一个视频链接或网页,输出一份带 YAML frontmatter 的结构化 Obsidian 笔记。本版本是首个公开预发布版,核心流程已可用,接口细节在 1.0.0 前可能调整。
|
|
9
|
+
|
|
10
|
+
## 新增
|
|
11
|
+
|
|
12
|
+
- **四种输入解析器插件**(`parsers/`):书籍(PDF/EPUB/MOBI)、论文(arXiv/PDF/HTML)、视频(yt-dlp 字幕抓取)、网页正文——每种可单独替换,fork 后只改自己的解析器
|
|
13
|
+
- **三波次 MapReduce 精读**:解析器获取分块 → 并行子代理精读(Map)→ 合并成稿(Reduce),长内容不溢出上下文
|
|
14
|
+
- **JSON Schema 结构化输出**(`schemas/`):各子任务结果受 schema 约束,不合格自动重试
|
|
15
|
+
- **幂等缓存**(`lib/cache.js`):对输入指纹去重,已处理过的输入直接跳过,省 token
|
|
16
|
+
- **失败分级**:配置错误直接终止(FATAL);内容解析失败降级为缺口标记继续(可降级)
|
|
17
|
+
- **质量校验**:关键引用标注出处 + 成稿自检 + 可选重试(`maxRetries`)
|
|
18
|
+
- **Obsidian 输出**:YAML frontmatter(type 字段可配合 Dataview 查询),输出目录可配置(`outputDir`)
|
|
19
|
+
- **工程配套**:CI(lint + 17 项 fixture 测试 + Gitleaks 密钥扫描)、SECURITY.md、CONTRIBUTING.md、CHANGELOG.md
|
|
20
|
+
|
|
21
|
+
## 修复
|
|
22
|
+
|
|
23
|
+
- 长文本获取不再依赖单次上下文(写入临时文件 + 行范围分块)
|
|
24
|
+
- 独立质量校验子代理超时问题(合并进成稿波次)
|
|
25
|
+
- 硬编码本地路径全部参数化(`outputDir` / `tempDir`),通过 6/6 安全检查
|
|
26
|
+
|
|
27
|
+
## 升级提醒
|
|
28
|
+
|
|
29
|
+
- 首个公开版本,无旧版迁移负担
|
|
30
|
+
- `options` 参数与 README「用法」示例保持一致即可
|
|
31
|
+
|
|
32
|
+
## 已知问题
|
|
33
|
+
|
|
34
|
+
- DSH 为 developer preview,升级后需重跑 `npm test` 验证
|
|
35
|
+
- 视频字幕依赖 yt-dlp,未安装时需手动提供转写文本
|
|
36
|
+
- 子代理写文件可能受权限限制,workflow 会返回内容由主代理落盘
|
|
37
|
+
|
|
38
|
+
## 验证
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
npm test # 17 项离线测试(不依赖真实 API)
|
|
42
|
+
npm run lint # 13 个 JS 文件语法检查
|
|
43
|
+
npm run validate
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
**当前功能列表(v0.1.0)**
|
|
47
|
+
|
|
48
|
+
| 能力 | 状态 |
|
|
49
|
+
|------|------|
|
|
50
|
+
| 书籍解析器(PDF/EPUB/MOBI) | ✅ |
|
|
51
|
+
| 论文解析器(arXiv/PDF/HTML) | ✅ |
|
|
52
|
+
| 视频解析器(yt-dlp 字幕) | ✅ |
|
|
53
|
+
| 网页解析器 | ✅ |
|
|
54
|
+
| 分块 + 并行精读(MapReduce) | ✅ |
|
|
55
|
+
| JSON Schema 输出约束 | ✅ |
|
|
56
|
+
| 幂等缓存 | ✅ |
|
|
57
|
+
| 失败分级 | ✅ |
|
|
58
|
+
| 质量校验循环 | ✅ |
|
|
59
|
+
| Obsidian 输出(Dataview 兼容) | ✅ |
|
|
60
|
+
| CI + 密钥扫描 | ✅ |
|
|
61
|
+
| 自定义解析器插件(custom-parsers/) | ✅ |
|
package/docs/RELEASE.md
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Release 模板
|
|
2
|
+
|
|
3
|
+
发布新版本时,把下面内容填好贴到 GitHub Releases 的 Release Notes 里。
|
|
4
|
+
不要直接抄 commit message("fix bug"、"update deps" 没人看得懂),要写**结果**:修了什么场景的问题、新增能力解决什么需求、升级要不要改配置。
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## vX.Y.Z(版本号规则:主版本=不兼容变更,次版本=新功能,修订号=修复)
|
|
9
|
+
|
|
10
|
+
**锁定的 DeepSeek Harness 版本**:`<版本号或 commit>`
|
|
11
|
+
**兼容性变化**:<有破坏性变更?是否影响已有配置?能否回退?>
|
|
12
|
+
|
|
13
|
+
### 新增
|
|
14
|
+
|
|
15
|
+
- <新功能,说明解决什么需求>
|
|
16
|
+
- ...
|
|
17
|
+
|
|
18
|
+
### 修复
|
|
19
|
+
|
|
20
|
+
- <修复了什么场景下的问题>
|
|
21
|
+
- ...
|
|
22
|
+
|
|
23
|
+
### 升级提醒
|
|
24
|
+
|
|
25
|
+
- <升级后要不要改配置 / 输入格式 / 输出路径>
|
|
26
|
+
- ...
|
|
27
|
+
|
|
28
|
+
### 已知问题
|
|
29
|
+
|
|
30
|
+
- <未解决的问题或限制>
|
|
31
|
+
- ...
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## 当前版本速览
|
|
36
|
+
|
|
37
|
+
| 版本 | 状态 | 说明 |
|
|
38
|
+
|------|------|------|
|
|
39
|
+
| v0.1.x | pre-release | 早期迭代,接口可能变 |
|
|
40
|
+
| v1.0.0 | 首个稳定版 | API 稳定后才发 |
|
|
41
|
+
|
|
42
|
+
> 早期迭代用 0.x 表达不稳定;API 稳定后再发 1.0.0。
|
|
43
|
+
> Tag 发布后保持稳定,出问题发修复版,不要反复改同一个 Tag。
|
package/index.js
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// deep-read-summarize — DSH plugin entry
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 导出 workflow (meta + script) 和解析器注册表,供 dsh 加载器识别。
|
|
5
|
+
// 安装方式(dsh profile 目录): pnpm add ./xxx.tgz → dsh.profile.bundles 追加
|
|
6
|
+
// ============================================================================
|
|
7
|
+
|
|
8
|
+
const workflow = require('./workflow');
|
|
9
|
+
const parsers = require('./parsers');
|
|
10
|
+
const schemas = require('./schemas');
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* 插件主入口:返回 dsh 可加载的插件定义
|
|
14
|
+
* - name: 插件名(对应 cordis.patch.yml 的 id)
|
|
15
|
+
* - meta: workflow 元信息(name/description/whenToUse/phases)
|
|
16
|
+
* - script: workflow 脚本(供 workflow 工具执行)
|
|
17
|
+
* - parsers: 解析器注册表(resolve/list)
|
|
18
|
+
* - schemas: JSON Schema 约束
|
|
19
|
+
*/
|
|
20
|
+
function load() {
|
|
21
|
+
return {
|
|
22
|
+
name: 'deep-read-summarize',
|
|
23
|
+
workflow: workflow,
|
|
24
|
+
parsers: parsers,
|
|
25
|
+
schemas: schemas
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
module.exports = { load, workflow, parsers, schemas };
|
|
30
|
+
|
|
31
|
+
// 兼容直接 require:插件系统可能直接读取 exports
|
|
32
|
+
module.exports.default = load;
|
package/lib/cache.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// lib/cache.js — idempotency: input fingerprint cache
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 对输入内容计算指纹(URL 或文件路径 + 文件大小/修改时间),
|
|
5
|
+
// 已处理过的输入直接跳过,避免重复消耗 token。
|
|
6
|
+
// 指纹记录以 JSONL 追加式存储,支持断点恢复。
|
|
7
|
+
// ============================================================================
|
|
8
|
+
// 用法:
|
|
9
|
+
// const cache = require("./lib/cache");
|
|
10
|
+
// const key = cache.fingerprint(input, fileMeta);
|
|
11
|
+
// if (cache.hasProcessed(cacheFile, key)) { skip; }
|
|
12
|
+
// cache.markProcessed(cacheFile, key, outputPath);
|
|
13
|
+
// ============================================================================
|
|
14
|
+
|
|
15
|
+
const fs = require("fs");
|
|
16
|
+
const path = require("path");
|
|
17
|
+
const crypto = require("crypto");
|
|
18
|
+
|
|
19
|
+
/** 计算输入指纹:URL 直接 hash;本地文件加大小 + 修改时间 */
|
|
20
|
+
function fingerprint(input, fileMeta) {
|
|
21
|
+
const base = typeof input === "string" ? input.trim() : "";
|
|
22
|
+
if (!base) return null;
|
|
23
|
+
const metaStr = fileMeta
|
|
24
|
+
? "|" + (fileMeta.size || "") + "|" + (fileMeta.mtimeMs || "")
|
|
25
|
+
: "";
|
|
26
|
+
return crypto.createHash("sha256").update(base + metaStr).digest("hex").slice(0, 16);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** 从缓存文件读取已处理记录(JSONL) */
|
|
30
|
+
function readCache(cacheFile) {
|
|
31
|
+
if (!fs.existsSync(cacheFile)) return new Map();
|
|
32
|
+
const map = new Map();
|
|
33
|
+
try {
|
|
34
|
+
const lines = fs.readFileSync(cacheFile, "utf8").split("\n");
|
|
35
|
+
for (const line of lines) {
|
|
36
|
+
if (!line.trim()) continue;
|
|
37
|
+
try {
|
|
38
|
+
const rec = JSON.parse(line);
|
|
39
|
+
if (rec && rec.key) map.set(rec.key, rec);
|
|
40
|
+
} catch (e) { /* 跳过损坏行 */ }
|
|
41
|
+
}
|
|
42
|
+
} catch (e) { /* 缓存不可读则视为空 */ }
|
|
43
|
+
return map;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** 是否已处理过该指纹 */
|
|
47
|
+
function hasProcessed(cacheFile, key) {
|
|
48
|
+
if (!key) return false;
|
|
49
|
+
return readCache(cacheFile).has(key);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** 标记已处理(追加 JSONL 记录) */
|
|
53
|
+
function markProcessed(cacheFile, key, outputPath, meta) {
|
|
54
|
+
if (!key) return false;
|
|
55
|
+
const dir = path.dirname(cacheFile);
|
|
56
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
57
|
+
const rec = { key, outputPath, ts: new Date().toISOString(), ...(meta || {}) };
|
|
58
|
+
fs.appendFileSync(cacheFile, JSON.stringify(rec) + "\n", "utf8");
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
module.exports = { fingerprint, readCache, hasProcessed, markProcessed };
|
package/package.json
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "deep-read-summarize",
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Deep reading & summarization for DSH: books/papers/videos/web → structured Obsidian notes. Plugin parsers, MapReduce deep-read, JSON Schema output, idempotent cache.",
|
|
5
|
+
"type": "commonjs",
|
|
6
|
+
"main": "index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./index.js",
|
|
9
|
+
"./workflow": "./workflow.js",
|
|
10
|
+
"./parsers": "./parsers/index.js",
|
|
11
|
+
"./schemas": "./schemas/index.js",
|
|
12
|
+
"./package.json": "./package.json"
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"index.js",
|
|
16
|
+
"workflow.js",
|
|
17
|
+
"parsers",
|
|
18
|
+
"schemas",
|
|
19
|
+
"lib",
|
|
20
|
+
"skills",
|
|
21
|
+
"scripts",
|
|
22
|
+
"tests",
|
|
23
|
+
"docs",
|
|
24
|
+
"cordis.patch.yml",
|
|
25
|
+
"CHANGELOG.md",
|
|
26
|
+
"README.md",
|
|
27
|
+
"LICENSE",
|
|
28
|
+
"SECURITY.md",
|
|
29
|
+
".env.example"
|
|
30
|
+
],
|
|
31
|
+
"scripts": {
|
|
32
|
+
"test": "node tests/run-tests.js",
|
|
33
|
+
"lint": "node scripts/lint.js",
|
|
34
|
+
"validate": "node tests/quick_validate.js && node scripts/security-check.js",
|
|
35
|
+
"security": "node scripts/security-check.js",
|
|
36
|
+
"test:node": "node --test tests/",
|
|
37
|
+
"prepublishOnly": "node tests/run-tests.js && node scripts/lint.js && node scripts/security-check.js"
|
|
38
|
+
},
|
|
39
|
+
"dsh": {
|
|
40
|
+
"bundle": {
|
|
41
|
+
"patch": "./cordis.patch.yml"
|
|
42
|
+
},
|
|
43
|
+
"lockedRuntime": "workflow tool semantics — see CHANGELOG.md",
|
|
44
|
+
"minVersion": "0.1.0",
|
|
45
|
+
"breakingPolicy": "re-validate with npm test after each DSH upgrade"
|
|
46
|
+
},
|
|
47
|
+
"keywords": [
|
|
48
|
+
"dsh",
|
|
49
|
+
"workflow",
|
|
50
|
+
"summarization",
|
|
51
|
+
"deep-read",
|
|
52
|
+
"obsidian",
|
|
53
|
+
"llm",
|
|
54
|
+
"subagent",
|
|
55
|
+
"mapreduce",
|
|
56
|
+
"parsers"
|
|
57
|
+
],
|
|
58
|
+
"license": "MIT",
|
|
59
|
+
"engines": {
|
|
60
|
+
"node": ">=18"
|
|
61
|
+
},
|
|
62
|
+
"repository": {
|
|
63
|
+
"type": "git",
|
|
64
|
+
"url": "https://github.com/PensiveFei/deep-read-summarize.git"
|
|
65
|
+
}
|
|
66
|
+
}
|