deep-read-summarize 0.3.14 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +13 -12
- package/CHANGELOG.md +417 -347
- package/README.en.md +37 -3
- package/README.md +36 -3
- package/cordis.patch.yml +3 -0
- package/docs/RELEASE.md +3 -1
- package/index.js +5 -3
- package/lib/cache.js +102 -102
- package/lib/workflow-script.txt +243 -0
- package/package.json +71 -73
- package/parsers/_prompt.js +44 -44
- package/scripts/lint.js +62 -14
- package/skills/deep-read-summarize/SKILL.md +8 -1
- package/workflow.js +6 -1
- package/docs/RELEASE-v0.1.0.md +0 -61
- package/tests/cache.test.js +0 -117
- package/tests/fixtures/fable.txt +0 -28
- package/tests/index.test.js +0 -518
- package/tests/quick_validate.js +0 -52
- package/tests/run-tests.js +0 -275
package/README.en.md
CHANGED
|
@@ -78,11 +78,17 @@ parsers/ Parsers for each input type, dispatched by type
|
|
|
78
78
|
web.js Web pages: main-text extraction
|
|
79
79
|
index.js Registry: parser discovery and fallback
|
|
80
80
|
schemas/ JSON Schema for the output contract (injected into the script by workflow.js - single source of truth)
|
|
81
|
-
scripts/ lint, security checks, cross-platform transcription bootstrap
|
|
82
|
-
|
|
83
|
-
|
|
81
|
+
scripts/ lint (incl. line-ending/whitespace hygiene), security checks, cross-platform transcription bootstrap
|
|
82
|
+
lib/ Small standalone helpers
|
|
83
|
+
cache.js input fingerprint and idempotency records (for callers driving the workflow themselves -- the
|
|
84
|
+
workflow sandbox has no filesystem, so it cannot use this)
|
|
85
|
+
workflow-script.txt the workflow script body (plain text: read and edit it directly, no escaping involved)
|
|
86
|
+
tests/ tests and validation scripts (**not published to npm**, repo only)
|
|
87
|
+
workflow.js the workflow meta and assembly logic (parser inlining, schema injection)
|
|
84
88
|
```
|
|
85
89
|
|
|
90
|
+
> The script body lives in `lib/workflow-script.txt` on purpose: this file was corrupted twice by "escaping on top of escaping". It is plain text now, so reading and editing it needs no encoding step.
|
|
91
|
+
|
|
86
92
|
To change how an input type is handled, drop a parser with the same interface into `custom-parsers/`; a type with the same name overrides the built-in implementation, and you can also add a brand-new type name (e.g. `podcast`) - every registered parser is inlined into the workflow script, and both the `type` whitelist and the schema's `kind` enum follow the registry. The interface has only three fields: `name`, `types`, `buildPrompt(input, opts)`.
|
|
87
93
|
|
|
88
94
|
---
|
|
@@ -190,6 +196,7 @@ Pass this JSON to DSH's workflow tool:
|
|
|
190
196
|
"language": "zh", // video ASR language: use "en" for English-only videos
|
|
191
197
|
"device": "cpu", // ASR device: cpu (default) | auto | cuda; auto/cuda needs a CUDA runtime and falls back to cpu
|
|
192
198
|
"maxRetries": 0, // quality-check retries (default 0 = no retry; retrying is noticeably slower)
|
|
199
|
+
"frontmatter": "obsidian", // key set: obsidian (default, for Dataview) | generic (drops the keys Hugo reserves) | minimal
|
|
193
200
|
"requireCitations": true, // whether key conclusions must carry citations
|
|
194
201
|
"includeTimestamps": false,
|
|
195
202
|
"outputDir": "./output", // note output directory (can point at your Obsidian vault or a static-site content folder)
|
|
@@ -209,6 +216,33 @@ Examples:
|
|
|
209
216
|
|
|
210
217
|
Notes are written to `options.outputDir` (default `./output`, can point at your Obsidian vault or a static-site content folder), with the filename taken from the content title.
|
|
211
218
|
|
|
219
|
+
### Frontmatter profiles (`options.frontmatter`)
|
|
220
|
+
|
|
221
|
+
The default is `obsidian`, byte-for-byte what earlier versions produced. Profiles exist because `type`, `aliases` and `url` are **reserved keys in Hugo** with different meanings -- dropping an Obsidian note into Hugo changes how the page renders, rather than adding a few harmless fields.
|
|
222
|
+
|
|
223
|
+
| Key | `obsidian` (default) | `generic` | `minimal` |
|
|
224
|
+
|---|---|---|---|
|
|
225
|
+
| `title` | yes | yes | yes |
|
|
226
|
+
| `aliases` | yes | -- | -- |
|
|
227
|
+
| `author` / `year` | yes | yes | -- |
|
|
228
|
+
| `type` | yes | -- | -- |
|
|
229
|
+
| `docType` | -- | yes | -- |
|
|
230
|
+
| `url` | yes | -- | -- |
|
|
231
|
+
| `sourceUrl` | -- | yes | yes |
|
|
232
|
+
| `tags` | yes | yes | yes |
|
|
233
|
+
| `created` | yes | -- | -- |
|
|
234
|
+
| `date` | -- | yes | yes |
|
|
235
|
+
| `status` | yes | yes | -- |
|
|
236
|
+
|
|
237
|
+
Why `generic` renames things:
|
|
238
|
+
|
|
239
|
+
- `type` in Hugo selects the **layout** (`type: video` makes it look for `layouts/video/`)
|
|
240
|
+
- `aliases` in Hugo is **a list of old URLs that redirect to this page**
|
|
241
|
+
- `url` in Hugo **overrides the page's own path** -- the source link would become the article's permalink
|
|
242
|
+
- `date` is the time key both Hugo and Jekyll understand; `created` is just a custom field
|
|
243
|
+
|
|
244
|
+
An unknown value is fatal rather than silently ignored, matching how `type` is handled.
|
|
245
|
+
|
|
212
246
|
### Video: full transcript (subtitles first, auto-transcription when none)
|
|
213
247
|
|
|
214
248
|
Video text extraction is a **unified pipeline** (the old "three tiers" are gone): the goal is to get the **full transcript**, then deep-read it.
|
package/README.md
CHANGED
|
@@ -78,11 +78,16 @@ parsers/ 各输入类型的解析器,按类型分发
|
|
|
78
78
|
web.js 网页:正文提取
|
|
79
79
|
index.js 注册表:解析器发现与回退
|
|
80
80
|
schemas/ 输出约束的 JSON Schema(波次1 由 workflow.js 注入脚本,单一真相)
|
|
81
|
-
scripts/ lint
|
|
82
|
-
|
|
83
|
-
|
|
81
|
+
scripts/ lint(含换行/空白卫生检查)、安全检查、跨平台转写引导(transcribe.js)
|
|
82
|
+
lib/ 独立可用的小工具
|
|
83
|
+
cache.js 输入指纹与幂等记录(供自己驱动流程的调用方使用;workflow 沙箱没有文件系统,用不到它)
|
|
84
|
+
workflow-script.txt workflow 脚本正文(普通文本,可直接读改,不需要任何转义)
|
|
85
|
+
tests/ 测试与验证脚本(**不随 npm 包发布**,仓库里才有)
|
|
86
|
+
workflow.js workflow 的 meta 与组装逻辑(解析器内联、schema 注入)
|
|
84
87
|
```
|
|
85
88
|
|
|
89
|
+
> 脚本正文单独放 `lib/workflow-script.txt`:这个文件曾经因为「转义套转义」被改坏过两次,现在它是普通文本,读改都不需要编码步骤。
|
|
90
|
+
|
|
86
91
|
想换某个输入类型的处理方式,在 `custom-parsers/` 放一个同接口的解析器即可,同名类型会覆盖内置实现;用新类型名(如 `podcast`)也可以——注册表里的解析器会全部内联进 workflow 脚本,`type` 白名单与 schema 的 `kind` 枚举都跟着注册表走。接口只有三个字段:`name`、`types`、`buildPrompt(input, opts)`。
|
|
87
92
|
|
|
88
93
|
---
|
|
@@ -190,6 +195,7 @@ parsers: book, paper, video, web
|
|
|
190
195
|
"language": "zh", // 视频转写语言:纯英文视频可设 "en"
|
|
191
196
|
"device": "cpu", // 转写设备:cpu(默认)| auto | cuda;auto/cuda 需自备 CUDA 运行库,失败自动回退 cpu
|
|
192
197
|
"maxRetries": 0, // 质量校验重试次数(默认 0 = 不重试,重试会明显变慢)
|
|
198
|
+
"frontmatter": "obsidian", // frontmatter 键集:obsidian(默认,配 Dataview)| generic(换掉 Hugo 保留键)| minimal
|
|
193
199
|
"requireCitations": true, // 关键结论是否必须标注出处
|
|
194
200
|
"includeTimestamps": false,
|
|
195
201
|
"outputDir": "./output", // 笔记输出目录(可指向 Obsidian 仓库或静态站点内容目录)
|
|
@@ -209,6 +215,33 @@ parsers: book, paper, video, web
|
|
|
209
215
|
|
|
210
216
|
笔记写到 `options.outputDir` 指定的目录(默认 `./output`,可指向你的 Obsidian 仓库或静态站点内容目录),文件名取自内容标题。
|
|
211
217
|
|
|
218
|
+
### frontmatter 三档(`options.frontmatter`)
|
|
219
|
+
|
|
220
|
+
默认 `obsidian`,与历史版本**逐字一致**。之所以要分档:`type`、`aliases`、`url` 在 Hugo 里是**保留键**,语义和 Obsidian 完全不同——直接丢进 Hugo 会改变渲染行为,而不是多几个无害字段。
|
|
221
|
+
|
|
222
|
+
| 键 | `obsidian`(默认) | `generic` | `minimal` |
|
|
223
|
+
|---|---|---|---|
|
|
224
|
+
| `title` | ✓ | ✓ | ✓ |
|
|
225
|
+
| `aliases` | ✓ | — | — |
|
|
226
|
+
| `author` / `year` | ✓ | ✓ | — |
|
|
227
|
+
| `type` | ✓ | — | — |
|
|
228
|
+
| `docType` | — | ✓ | — |
|
|
229
|
+
| `url` | ✓ | — | — |
|
|
230
|
+
| `sourceUrl` | — | ✓ | ✓ |
|
|
231
|
+
| `tags` | ✓ | ✓ | ✓ |
|
|
232
|
+
| `created` | ✓ | — | — |
|
|
233
|
+
| `date` | — | ✓ | ✓ |
|
|
234
|
+
| `status` | ✓ | ✓ | — |
|
|
235
|
+
|
|
236
|
+
为什么 `generic` 要换键:
|
|
237
|
+
|
|
238
|
+
- `type` 在 Hugo 里决定**用哪套 layout**(`type: video` 会让它去找 `layouts/video/`)
|
|
239
|
+
- `aliases` 在 Hugo 里是**一组会重定向到本页的旧 URL**
|
|
240
|
+
- `url` 在 Hugo 里会**覆盖页面自身的路径**——来源链接会变成文章的永久链接
|
|
241
|
+
- `date` 才是 Hugo/Jekyll 都认的时间键;`created` 只是自定义字段
|
|
242
|
+
|
|
243
|
+
非法值直接 FATAL 终止(与 `type` 的失败纪律一致,不静默降级)。
|
|
244
|
+
|
|
212
245
|
### 视频:完整逐字稿(字幕优先,无字幕自动转写)
|
|
213
246
|
|
|
214
247
|
视频取文本是**统一流水线**(已去掉「三档」):目标 = 拿到**完整逐字稿**再精读。
|
package/cordis.patch.yml
CHANGED
|
@@ -26,6 +26,9 @@
|
|
|
26
26
|
# language: 'zh' # transcription language (default: zh)
|
|
27
27
|
# device: 'cpu' # cpu | auto | cuda; GPU needs a CUDA runtime (default: cpu)
|
|
28
28
|
# maxRetries: 0 # quality-check retry rounds (default: 0)
|
|
29
|
+
# frontmatter: 'obsidian' # obsidian | generic | minimal (default: obsidian)
|
|
30
|
+
# # generic swaps the keys Hugo reserves: type -> docType,
|
|
31
|
+
# # url -> sourceUrl, created -> date; minimal keeps four keys
|
|
29
32
|
#
|
|
30
33
|
# Idempotency is deliberately NOT a plugin config key: the workflow script runs
|
|
31
34
|
# without a filesystem, so it cannot write a fingerprint record. Programmatic
|
package/docs/RELEASE.md
CHANGED
|
@@ -115,7 +115,9 @@ npm view deep-read-summarize --prefer-online # 验证线
|
|
|
115
115
|
| v0.3.11 | npm 已发布 | 文档:README 补「支持平台」;CI:三平台跑全套门禁 |
|
|
116
116
|
| v0.3.12 | npm 已发布 | 文档:定位改为「标准 Markdown」;删 dsh.so 一节;macOS 真机验证转写通过 |
|
|
117
117
|
| v0.3.13 | npm 已发布 | 声明:补 `engines.dsh`(此前的 `dsh.minVersion` 非公开字段,市场不读,判定一直是 `undeclared`) |
|
|
118
|
-
|
|
|
118
|
+
| v0.3.14 | npm 已发布 | 修:路径尾分隔符、本地日期、config 白名单;README 测试数 |
|
|
119
|
+
| v0.3.15 | npm 已发布 | 工程卫生:锁定 LF、脚本正文独立成文、lint 扩检查、包瘦身、提示词改「标准 Markdown」 |
|
|
120
|
+
| **v0.3.16** | **npm 已发布(当前)** | 新增 `options.frontmatter` 三档(obsidian / generic / minimal),解决 Hugo 保留键冲突;默认档不变 |
|
|
119
121
|
| v1.0.0 | 计划 | API 稳定后发布 |
|
|
120
122
|
|
|
121
123
|
> 早期迭代用 0.x 表达不稳定;API 稳定后再发 1.0.0。
|
package/index.js
CHANGED
|
@@ -37,7 +37,8 @@ const DEFAULT_OPTIONS = {
|
|
|
37
37
|
whisperModel: 'small',
|
|
38
38
|
language: 'zh',
|
|
39
39
|
device: 'cpu',
|
|
40
|
-
maxRetries: 0
|
|
40
|
+
maxRetries: 0,
|
|
41
|
+
frontmatter: 'obsidian'
|
|
41
42
|
};
|
|
42
43
|
|
|
43
44
|
const OPTION_TYPES = {
|
|
@@ -52,7 +53,8 @@ const OPTION_TYPES = {
|
|
|
52
53
|
whisperModel: 'string',
|
|
53
54
|
language: 'string',
|
|
54
55
|
device: 'string',
|
|
55
|
-
maxRetries: 'number'
|
|
56
|
+
maxRetries: 'number',
|
|
57
|
+
frontmatter: 'string'
|
|
56
58
|
};
|
|
57
59
|
|
|
58
60
|
/**
|
|
@@ -73,7 +75,7 @@ function normalizeOptions(config) {
|
|
|
73
75
|
opts[key] = v;
|
|
74
76
|
}
|
|
75
77
|
// 未知键此前是完全无声的:循环只遍历白名单,写错的键名既不生效也不告警。
|
|
76
|
-
|
|
78
|
+
const unknown = Object.keys(raw).filter(function (k) {
|
|
77
79
|
return !Object.prototype.hasOwnProperty.call(DEFAULT_OPTIONS, k);
|
|
78
80
|
});
|
|
79
81
|
if (unknown.length) {
|
package/lib/cache.js
CHANGED
|
@@ -1,102 +1,102 @@
|
|
|
1
|
-
// ============================================================================
|
|
2
|
-
// lib/cache.js — idempotency: input fingerprint cache
|
|
3
|
-
// ============================================================================
|
|
4
|
-
// 对输入内容计算指纹(URL 或文件路径 + 文件大小/修改时间),
|
|
5
|
-
// 已处理过的输入直接跳过,避免重复消耗 token。
|
|
6
|
-
// 指纹记录以 JSONL 追加式存储,支持断点恢复。
|
|
7
|
-
// ============================================================================
|
|
8
|
-
// 用法:
|
|
9
|
-
// const cache = require("./lib/cache");
|
|
10
|
-
// const key = cache.fingerprint(input, fileMeta);
|
|
11
|
-
// if (cache.hasProcessed(cacheFile, key)) { skip; }
|
|
12
|
-
// cache.markProcessed(cacheFile, key, outputPath);
|
|
13
|
-
// ============================================================================
|
|
14
|
-
|
|
15
|
-
const fs = require("fs");
|
|
16
|
-
const path = require("path");
|
|
17
|
-
const crypto = require("crypto");
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* 规范化输入:对 http(s) URL 做归一化(host 小写、去默认端口、去 fragment、
|
|
21
|
-
* 去空 query 的 "?"、去尾部斜杠),避免同一 URL 的不同写法产生不同指纹;
|
|
22
|
-
* 非 URL(本地路径/纯文本)原样返回。
|
|
23
|
-
*/
|
|
24
|
-
function normalizeInput(input) {
|
|
25
|
-
const s = typeof input === "string" ? input.trim() : "";
|
|
26
|
-
if (!/^https?:\/\//i.test(s)) return s;
|
|
27
|
-
try {
|
|
28
|
-
const url = new URL(s);
|
|
29
|
-
url.hash = "";
|
|
30
|
-
url.hostname = url.hostname.toLowerCase();
|
|
31
|
-
if ((url.protocol === "http:" && url.port === "80") || (url.protocol === "https:" && url.port === "443")) {
|
|
32
|
-
url.port = "";
|
|
33
|
-
}
|
|
34
|
-
if (url.pathname.length > 1 && url.pathname.endsWith("/")) {
|
|
35
|
-
url.pathname = url.pathname.slice(0, -1);
|
|
36
|
-
}
|
|
37
|
-
// 去掉 URL 尾部的空 query "?"(new URL("...doc?").search 返回 "",无法靠 search 判断,
|
|
38
|
-
// 需检查 toString() 输出;同一 URL 的 "doc" 与 "doc?" 两种写法应共享指纹)
|
|
39
|
-
let normalized = url.toString();
|
|
40
|
-
if (normalized.endsWith("?")) normalized = normalized.slice(0, -1);
|
|
41
|
-
return normalized;
|
|
42
|
-
} catch (e) {
|
|
43
|
-
return s; // 非法 URL 原样处理
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/** 计算输入指纹:URL 规范化后 hash;本地文件加大小 + 修改时间 */
|
|
48
|
-
function fingerprint(input, fileMeta) {
|
|
49
|
-
const base = normalizeInput(input);
|
|
50
|
-
if (!base) return null;
|
|
51
|
-
const metaStr = fileMeta
|
|
52
|
-
? "|" + (fileMeta.size ?? "") + "|" + (fileMeta.mtimeMs ?? "")
|
|
53
|
-
: "";
|
|
54
|
-
return crypto.createHash("sha256").update(base + metaStr).digest("hex").slice(0, 16);
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/** 从缓存文件读取已处理记录(JSONL) */
|
|
58
|
-
function readCache(cacheFile) {
|
|
59
|
-
if (!fs.existsSync(cacheFile)) return new Map();
|
|
60
|
-
const map = new Map();
|
|
61
|
-
try {
|
|
62
|
-
const lines = fs.readFileSync(cacheFile, "utf8").split("\n");
|
|
63
|
-
for (const line of lines) {
|
|
64
|
-
if (!line.trim()) continue;
|
|
65
|
-
try {
|
|
66
|
-
const rec = JSON.parse(line);
|
|
67
|
-
if (rec && rec.key) map.set(rec.key, rec);
|
|
68
|
-
} catch (e) { /* 跳过损坏行 */ }
|
|
69
|
-
}
|
|
70
|
-
} catch (e) { /* 缓存不可读则视为空 */ }
|
|
71
|
-
return map;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/** 是否已处理过该指纹;opts.maxAgeMs 提供时,过期(记录时间早于阈值)视为未处理 */
|
|
75
|
-
function hasProcessed(cacheFile, key, opts) {
|
|
76
|
-
if (!key) return false;
|
|
77
|
-
const rec = readCache(cacheFile).get(key);
|
|
78
|
-
if (!rec) return false;
|
|
79
|
-
const maxAgeMs = opts && opts.maxAgeMs;
|
|
80
|
-
if (maxAgeMs && rec.ts) {
|
|
81
|
-
const age = Date.now() - new Date(rec.ts).getTime();
|
|
82
|
-
if (!isFinite(age) || age > maxAgeMs) return false;
|
|
83
|
-
}
|
|
84
|
-
return true;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
/** 标记已处理(追加 JSONL 记录);写缓存失败仅告警,不中断主流程 */
|
|
88
|
-
function markProcessed(cacheFile, key, outputPath, meta) {
|
|
89
|
-
if (!key) return false;
|
|
90
|
-
try {
|
|
91
|
-
const dir = path.dirname(cacheFile);
|
|
92
|
-
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
93
|
-
const rec = { key, outputPath, ts: new Date().toISOString(), ...(meta || {}) };
|
|
94
|
-
fs.appendFileSync(cacheFile, JSON.stringify(rec) + "\n", "utf8");
|
|
95
|
-
return true;
|
|
96
|
-
} catch (e) {
|
|
97
|
-
console.warn("[cache] markProcessed failed:", e.message);
|
|
98
|
-
return false;
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
module.exports = { fingerprint, normalizeInput, readCache, hasProcessed, markProcessed };
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// lib/cache.js — idempotency: input fingerprint cache
|
|
3
|
+
// ============================================================================
|
|
4
|
+
// 对输入内容计算指纹(URL 或文件路径 + 文件大小/修改时间),
|
|
5
|
+
// 已处理过的输入直接跳过,避免重复消耗 token。
|
|
6
|
+
// 指纹记录以 JSONL 追加式存储,支持断点恢复。
|
|
7
|
+
// ============================================================================
|
|
8
|
+
// 用法:
|
|
9
|
+
// const cache = require("./lib/cache");
|
|
10
|
+
// const key = cache.fingerprint(input, fileMeta);
|
|
11
|
+
// if (cache.hasProcessed(cacheFile, key)) { skip; }
|
|
12
|
+
// cache.markProcessed(cacheFile, key, outputPath);
|
|
13
|
+
// ============================================================================
|
|
14
|
+
|
|
15
|
+
const fs = require("fs");
|
|
16
|
+
const path = require("path");
|
|
17
|
+
const crypto = require("crypto");
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* 规范化输入:对 http(s) URL 做归一化(host 小写、去默认端口、去 fragment、
|
|
21
|
+
* 去空 query 的 "?"、去尾部斜杠),避免同一 URL 的不同写法产生不同指纹;
|
|
22
|
+
* 非 URL(本地路径/纯文本)原样返回。
|
|
23
|
+
*/
|
|
24
|
+
function normalizeInput(input) {
|
|
25
|
+
const s = typeof input === "string" ? input.trim() : "";
|
|
26
|
+
if (!/^https?:\/\//i.test(s)) return s;
|
|
27
|
+
try {
|
|
28
|
+
const url = new URL(s);
|
|
29
|
+
url.hash = "";
|
|
30
|
+
url.hostname = url.hostname.toLowerCase();
|
|
31
|
+
if ((url.protocol === "http:" && url.port === "80") || (url.protocol === "https:" && url.port === "443")) {
|
|
32
|
+
url.port = "";
|
|
33
|
+
}
|
|
34
|
+
if (url.pathname.length > 1 && url.pathname.endsWith("/")) {
|
|
35
|
+
url.pathname = url.pathname.slice(0, -1);
|
|
36
|
+
}
|
|
37
|
+
// 去掉 URL 尾部的空 query "?"(new URL("...doc?").search 返回 "",无法靠 search 判断,
|
|
38
|
+
// 需检查 toString() 输出;同一 URL 的 "doc" 与 "doc?" 两种写法应共享指纹)
|
|
39
|
+
let normalized = url.toString();
|
|
40
|
+
if (normalized.endsWith("?")) normalized = normalized.slice(0, -1);
|
|
41
|
+
return normalized;
|
|
42
|
+
} catch (e) {
|
|
43
|
+
return s; // 非法 URL 原样处理
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** 计算输入指纹:URL 规范化后 hash;本地文件加大小 + 修改时间 */
|
|
48
|
+
function fingerprint(input, fileMeta) {
|
|
49
|
+
const base = normalizeInput(input);
|
|
50
|
+
if (!base) return null;
|
|
51
|
+
const metaStr = fileMeta
|
|
52
|
+
? "|" + (fileMeta.size ?? "") + "|" + (fileMeta.mtimeMs ?? "")
|
|
53
|
+
: "";
|
|
54
|
+
return crypto.createHash("sha256").update(base + metaStr).digest("hex").slice(0, 16);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** 从缓存文件读取已处理记录(JSONL) */
|
|
58
|
+
function readCache(cacheFile) {
|
|
59
|
+
if (!fs.existsSync(cacheFile)) return new Map();
|
|
60
|
+
const map = new Map();
|
|
61
|
+
try {
|
|
62
|
+
const lines = fs.readFileSync(cacheFile, "utf8").split("\n");
|
|
63
|
+
for (const line of lines) {
|
|
64
|
+
if (!line.trim()) continue;
|
|
65
|
+
try {
|
|
66
|
+
const rec = JSON.parse(line);
|
|
67
|
+
if (rec && rec.key) map.set(rec.key, rec);
|
|
68
|
+
} catch (e) { /* 跳过损坏行 */ }
|
|
69
|
+
}
|
|
70
|
+
} catch (e) { /* 缓存不可读则视为空 */ }
|
|
71
|
+
return map;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** 是否已处理过该指纹;opts.maxAgeMs 提供时,过期(记录时间早于阈值)视为未处理 */
|
|
75
|
+
function hasProcessed(cacheFile, key, opts) {
|
|
76
|
+
if (!key) return false;
|
|
77
|
+
const rec = readCache(cacheFile).get(key);
|
|
78
|
+
if (!rec) return false;
|
|
79
|
+
const maxAgeMs = opts && opts.maxAgeMs;
|
|
80
|
+
if (maxAgeMs && rec.ts) {
|
|
81
|
+
const age = Date.now() - new Date(rec.ts).getTime();
|
|
82
|
+
if (!isFinite(age) || age > maxAgeMs) return false;
|
|
83
|
+
}
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** 标记已处理(追加 JSONL 记录);写缓存失败仅告警,不中断主流程 */
|
|
88
|
+
function markProcessed(cacheFile, key, outputPath, meta) {
|
|
89
|
+
if (!key) return false;
|
|
90
|
+
try {
|
|
91
|
+
const dir = path.dirname(cacheFile);
|
|
92
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
93
|
+
const rec = { key, outputPath, ts: new Date().toISOString(), ...(meta || {}) };
|
|
94
|
+
fs.appendFileSync(cacheFile, JSON.stringify(rec) + "\n", "utf8");
|
|
95
|
+
return true;
|
|
96
|
+
} catch (e) {
|
|
97
|
+
console.warn("[cache] markProcessed failed:", e.message);
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
module.exports = { fingerprint, normalizeInput, readCache, hasProcessed, markProcessed };
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
// ========== deep-read-summarize v3.0 ==========
|
|
2
|
+
// 3 波次:解析器获取分块 → 并行精读 → 成稿+质量校验
|
|
3
|
+
// 特性:解析器插件化 / 失败纪律(fatal vs 降级) / 成本控制(maxChunks 强制,块数=子代理数)
|
|
4
|
+
|
|
5
|
+
// YAML 双引号标量:JSON.stringify 的转义规则是 YAML double-quoted 风格的子集,
|
|
6
|
+
// 标题/作者里出现引号时不会再截断 frontmatter(旧版是裸拼 "...")。
|
|
7
|
+
function yamlStr(v) { return JSON.stringify(v == null ? "" : String(v)); }
|
|
8
|
+
|
|
9
|
+
// ---------- 0. 配置校验(fatal 级错误:必须终止) ----------
|
|
10
|
+
const input = (args && args.input || "").trim();
|
|
11
|
+
const forcedType = ((args && args.type) || "auto").toLowerCase();
|
|
12
|
+
const options = (args && args.options) || {};
|
|
13
|
+
const minWords = options.minWords || 2500;
|
|
14
|
+
const fastMode = !!options.fastMode;
|
|
15
|
+
const maxChunks = Math.max(1, Math.min(options.maxChunks || 4, 12));
|
|
16
|
+
const requireCitations = options.requireCitations !== false;
|
|
17
|
+
const includeTimestamps = !!options.includeTimestamps;
|
|
18
|
+
const maxRetries = options.maxRetries || 0;
|
|
19
|
+
const outputDir = options.outputDir || "./output";
|
|
20
|
+
const tempDir = options.tempDir || "./.tmp";
|
|
21
|
+
const tempFile = tempDir.replace(/[\\/]+$/, "") + "/input.txt";
|
|
22
|
+
const whisperModel = options.whisperModel || "small";
|
|
23
|
+
const transcribeLanguage = options.language || "zh";
|
|
24
|
+
const transcribeDevice = options.device || "cpu";
|
|
25
|
+
// frontmatter 档位。默认 obsidian = 历史行为,逐字不变(现有用户的 Dataview 查询依赖 type / status)。
|
|
26
|
+
// generic 档换掉三个「Hugo 保留键」:type -> docType、url -> sourceUrl、created -> date(date 才是标准键);
|
|
27
|
+
// minimal 档只留 title / date / sourceUrl / tags。
|
|
28
|
+
const FM_PROFILES = ["obsidian", "generic", "minimal"];
|
|
29
|
+
const fmProfile = String(options.frontmatter || "obsidian").toLowerCase();
|
|
30
|
+
if (FM_PROFILES.indexOf(fmProfile) === -1) {
|
|
31
|
+
throw new Error("[FATAL] options.frontmatter 非法:" + options.frontmatter + "(可用:" + FM_PROFILES.join("|") + ")");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// 幂等不在这里做:脚本沙箱没有文件系统,写不了指纹记录。
|
|
35
|
+
// 需要幂等的调用方请用 lib/cache.js(fingerprint / hasProcessed / markProcessed)。
|
|
36
|
+
|
|
37
|
+
// ---------- 1. 解析器表:构建时把解析器内联为 __parsers(自包含);
|
|
38
|
+
// 宿主若注入 args._parsers 则优先使用 ----------
|
|
39
|
+
const parsers = args._parsers || {};
|
|
40
|
+
const parserNames = Object.keys(parsers);
|
|
41
|
+
|
|
42
|
+
// 配置校验依赖解析器表,必须放在其后
|
|
43
|
+
if (!input) throw new Error("[FATAL] args.input 缺失:请输入内容链接或文件路径");
|
|
44
|
+
if (parserNames.length === 0) throw new Error("[FATAL] 解析器表为空:脚本没有内联任何解析器");
|
|
45
|
+
if (forcedType !== "auto" && parserNames.indexOf(forcedType) === -1)
|
|
46
|
+
throw new Error("[FATAL] args.type 非法:" + forcedType + "(可用:" + ["auto"].concat(parserNames).join("|") + ")");
|
|
47
|
+
|
|
48
|
+
phase("解析器选择");
|
|
49
|
+
const kindNames = { book: "书籍", paper: "学术论文", video: "视频", web: "网页/文章" };
|
|
50
|
+
const kindFocus = {
|
|
51
|
+
book: "按章节主题展开核心论点、论证、关键例子、章节间关系,收集关键概念与可引用原文(含页码/章节)",
|
|
52
|
+
paper: "四维精读:1)研究问题与方法 2)结论与贡献 3)局限与批判 4)与同类工作比较;收集术语与可引用原文(含章节/段落)",
|
|
53
|
+
video: "按主题脉络组织,提取核心论点、证据、例子、可操作建议,收集关键概念与主讲人原话" + (includeTimestamps ? "(含时间戳定位)" : "(不标时间戳)"),
|
|
54
|
+
web: "提取核心论点、结构拆解、关键概念、可引用原文"
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
// ---------- 2. 波次1:解析器获取 + 分块 ----------
|
|
58
|
+
phase("获取与分块");
|
|
59
|
+
const fetchPrompt = (parsers[forcedType !== "auto" ? forcedType : "web"] || parsers.web || { buildPrompt: function() { return ["未知解析器"]; } })
|
|
60
|
+
.buildPrompt(input, { tempDir: tempDir, tempFile: tempFile, maxChunks: maxChunks, transcribe: options.transcribe !== false, whisperModel: whisperModel, language: transcribeLanguage, device: transcribeDevice });
|
|
61
|
+
// fetchSchema 由插件在构建时从 schemas/index.js 注入(单一真相,避免两份 schema 漂移)。
|
|
62
|
+
|
|
63
|
+
const fetched = await agent(fetchPrompt, { schema: fetchSchema, label: "获取+分块", phase: "获取与分块" });
|
|
64
|
+
|
|
65
|
+
// 失败纪律:内容获取失败 = 可降级错误(返回失败结果,不抛 fatal)
|
|
66
|
+
if (!fetched || !fetched.saved) {
|
|
67
|
+
return { ok: false, stage: "fetch", fatal: false, message: (fetched && fetched.message) || "获取失败(可降级:检查输入或手动提供文本)" };
|
|
68
|
+
}
|
|
69
|
+
// 分块计划收敛:块数 = 子代理数,计划失控会顶穿 workflow 单次调用时限;
|
|
70
|
+
// 同时挡掉倒置/越界的行号区间(原样透传会让 read 收到 limit<=0)。
|
|
71
|
+
const rawPlan = (fetched && fetched.chunkPlan) || [];
|
|
72
|
+
if (rawPlan.length === 0) return { ok: false, stage: "fetch", fatal: false, message: "分块计划为空" };
|
|
73
|
+
const declaredLines = Math.floor(Number(fetched.totalLines) || 0);
|
|
74
|
+
const chunkPlan = [];
|
|
75
|
+
for (let i = 0; i < rawPlan.length && chunkPlan.length < maxChunks; i++) {
|
|
76
|
+
const c = rawPlan[i] || {};
|
|
77
|
+
const s = Math.floor(Number(c.startLine) || 0);
|
|
78
|
+
const e = Math.floor(Number(c.endLine) || 0);
|
|
79
|
+
if (s < 1 || e < s) continue;
|
|
80
|
+
if (declaredLines > 0 && s > declaredLines) continue;
|
|
81
|
+
chunkPlan.push({
|
|
82
|
+
id: chunkPlan.length + 1,
|
|
83
|
+
topic: String(c.topic || ("块 " + (chunkPlan.length + 1))),
|
|
84
|
+
startLine: s,
|
|
85
|
+
endLine: declaredLines > 0 ? Math.min(e, declaredLines) : e
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
if (chunkPlan.length === 0) return { ok: false, stage: "fetch", fatal: false, message: "分块计划无效:行号区间缺失或非法" };
|
|
89
|
+
if (chunkPlan.length < rawPlan.length) log("分块计划收敛:" + rawPlan.length + " -> " + chunkPlan.length + " 块(上限 " + maxChunks + ")");
|
|
90
|
+
|
|
91
|
+
// 类型:auto 时只接受解析器表里真实存在的类型,否则退回回退解析器,
|
|
92
|
+
// 避免 "auto" 这种非法值流进 frontmatter 的 type / tags。
|
|
93
|
+
let kind = forcedType;
|
|
94
|
+
if (forcedType === "auto") {
|
|
95
|
+
kind = (typeof fetched.kind === "string" && parserNames.indexOf(fetched.kind) !== -1)
|
|
96
|
+
? fetched.kind
|
|
97
|
+
: (parserNames.indexOf("web") !== -1 ? "web" : parserNames[0]);
|
|
98
|
+
}
|
|
99
|
+
const metadata = (fetched && fetched.metadata) || {};
|
|
100
|
+
const textSource = (metadata && metadata.textSource) || "unknown";
|
|
101
|
+
const srcLabels = { desc: "视频简介/desc", subtitle: "平台字幕", transcription: "完整语音转写", manual: "用户提供转写文本" };
|
|
102
|
+
const srcLabel = srcLabels[textSource] || textSource;
|
|
103
|
+
log("[source] " + textSource + " -> " + srcLabel);
|
|
104
|
+
log("类型:" + kind + " 行数:" + fetched.totalLines + " 块数:" + chunkPlan.length);
|
|
105
|
+
|
|
106
|
+
// ---------- 3. 波次2:并行精读(Map) ----------
|
|
107
|
+
phase("并行精读");
|
|
108
|
+
const chunkPrompts = chunkPlan.map(function (c) {
|
|
109
|
+
return [
|
|
110
|
+
"请深度精读以下内容块,输出结构化精读笔记。",
|
|
111
|
+
"内容类型:" + (kindNames[kind] || kind) + " | 本块主题:" + c.topic,
|
|
112
|
+
"精读侧重:" + (kindFocus[kind] || kindFocus.web),
|
|
113
|
+
"用 read 工具读取 " + tempFile + " 第 " + c.startLine + " 到 " + c.endLine + " 行(offset=" + c.startLine + ", limit=" + (c.endLine - c.startLine + 1) + "),然后输出:",
|
|
114
|
+
"## 本块主题:" + c.topic,
|
|
115
|
+
"### 要点(详细展开,保留数据/例子/论证)",
|
|
116
|
+
"### 关键概念(术语+解释)",
|
|
117
|
+
"### 关键引用(最多2句原文+位置)" + (requireCitations ? ",必须标注页码/章节/段落位置" : ""),
|
|
118
|
+
"### 批判观察(如有)"
|
|
119
|
+
].join("\n");
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
const reads = [];
|
|
123
|
+
const readBatch = 3;
|
|
124
|
+
for (let bi = 0; bi < chunkPrompts.length; bi += readBatch) {
|
|
125
|
+
const batch = chunkPrompts.slice(bi, bi + readBatch);
|
|
126
|
+
const batchReads = await parallel(batch.map(function (p) {
|
|
127
|
+
return function () { return agent(p, { label: "精读块", phase: "并行精读" }); };
|
|
128
|
+
}));
|
|
129
|
+
for (let k = 0; k < batchReads.length; k++) reads.push(batchReads[k]);
|
|
130
|
+
}
|
|
131
|
+
const validReads = reads.filter(function (r) { return r && r.trim && r.trim().length > 0; });
|
|
132
|
+
|
|
133
|
+
// 失败纪律:某块精读失败 = 可降级(跳过该块,标记缺口继续)
|
|
134
|
+
const failedChunks = chunkPlan.length - validReads.length;
|
|
135
|
+
if (validReads.length === 0) return { ok: false, stage: "read", fatal: false, message: "所有分块精读均失败" };
|
|
136
|
+
if (failedChunks > 0) log("警告:" + failedChunks + " 块精读失败,标记缺口继续成稿");
|
|
137
|
+
log("精读完成:" + validReads.length + "/" + chunkPlan.length);
|
|
138
|
+
|
|
139
|
+
// ---------- 4. 波次3:成稿 + 内嵌质量自检(Reduce) ----------
|
|
140
|
+
phase("汇总成稿");
|
|
141
|
+
const todayDate = new Date();
|
|
142
|
+
const today = todayDate.getFullYear() + "-" + ("0" + (todayDate.getMonth() + 1)).slice(-2) + "-" + ("0" + todayDate.getDate()).slice(-2);
|
|
143
|
+
// 标题先折叠换行/制表与连续空白:否则会同时污染 .md 文件名和 frontmatter。
|
|
144
|
+
const finalTitle = String(metadata.title || "").replace(/[\r\n\t]+/g, " ").replace(/\s+/g, " ").trim() || "未命名内容";
|
|
145
|
+
// 文件名清洗:去掉 Windows 非法字符;全部非法时兜底,避免写出 ./output/.md 这种隐藏文件。
|
|
146
|
+
const safeTitle = finalTitle.replace(/[\\/:*?"<>|]/g, " ").replace(/\s+/g, " ").trim() || "未命名内容";
|
|
147
|
+
const mdPath = outputDir.replace(/[\\/]+$/, "") + "/" + safeTitle + ".md";
|
|
148
|
+
|
|
149
|
+
const extraSections = fastMode ? [] : [
|
|
150
|
+
"", "## 5. 批判性分析(论证强处/弱处/可质疑之处)", "",
|
|
151
|
+
"## 6. 个人思考与应用(与已有知识联系、如何应用、行动要点)", "",
|
|
152
|
+
"## 7. 延伸阅读(相关主题/作者其他作品/参考文献)", ""
|
|
153
|
+
];
|
|
154
|
+
|
|
155
|
+
// frontmatter 按档位组装。obsidian 档与历史版本逐字一致,不动既有用户的 Dataview 查询。
|
|
156
|
+
const fm = ["---"];
|
|
157
|
+
if (fmProfile === "obsidian") {
|
|
158
|
+
fm.push('title: ' + yamlStr("《" + finalTitle + "》深度精读笔记"));
|
|
159
|
+
fm.push("aliases: [" + yamlStr(finalTitle) + "]");
|
|
160
|
+
fm.push("author: " + yamlStr(metadata.author));
|
|
161
|
+
fm.push("year: " + yamlStr(metadata.year));
|
|
162
|
+
fm.push("type: " + yamlStr(kind));
|
|
163
|
+
fm.push("url: " + yamlStr(metadata.url || input));
|
|
164
|
+
fm.push("tags: [精读, " + (kindNames[kind] || kind) + "]");
|
|
165
|
+
fm.push("created: " + today);
|
|
166
|
+
fm.push("status: 已完成");
|
|
167
|
+
} else if (fmProfile === "generic") {
|
|
168
|
+
fm.push('title: ' + yamlStr("《" + finalTitle + "》深度精读笔记"));
|
|
169
|
+
fm.push("date: " + today);
|
|
170
|
+
fm.push("author: " + yamlStr(metadata.author));
|
|
171
|
+
fm.push("year: " + yamlStr(metadata.year));
|
|
172
|
+
fm.push("docType: " + yamlStr(kind));
|
|
173
|
+
fm.push("sourceUrl: " + yamlStr(metadata.url || input));
|
|
174
|
+
fm.push("tags: [精读, " + (kindNames[kind] || kind) + "]");
|
|
175
|
+
fm.push("status: 已完成");
|
|
176
|
+
} else {
|
|
177
|
+
fm.push('title: ' + yamlStr("《" + finalTitle + "》深度精读笔记"));
|
|
178
|
+
fm.push("date: " + today);
|
|
179
|
+
fm.push("sourceUrl: " + yamlStr(metadata.url || input));
|
|
180
|
+
fm.push("tags: [精读, " + (kindNames[kind] || kind) + "]");
|
|
181
|
+
}
|
|
182
|
+
fm.push("---");
|
|
183
|
+
|
|
184
|
+
const draftPrompt = [
|
|
185
|
+
"将以下多块精读结果整合为完整的结构化精读笔记,只输出标准 Markdown(含 YAML frontmatter)。",
|
|
186
|
+
"模板:"
|
|
187
|
+
].concat(fm, [
|
|
188
|
+
"# 《" + finalTitle + "》深度精读笔记", "",
|
|
189
|
+
"> 一句话核心概括。", "",
|
|
190
|
+
"## 1. 概述(约500字:主题/背景/核心问题/总体结论)", "",
|
|
191
|
+
"## 2. 结构拆解(按主题组织,含详细要点;关键结论标注原文出处)", "",
|
|
192
|
+
"## 3. 关键概念与术语表(表格)", "",
|
|
193
|
+
"## 4. 关键引用(仅关键结论" + (requireCitations ? ",必须标注页码/章节/段落" : "") + ")",
|
|
194
|
+
], extraSections, [
|
|
195
|
+
"---",
|
|
196
|
+
"*笔记整理于 " + today + "*", "",
|
|
197
|
+
"硬性要求:总篇幅≥" + minWords + "字;语言与原文一致;术语统一;引用必须真实;视频" + (includeTimestamps ? "标注时间戳" : "不标注时间戳") + "。",
|
|
198
|
+
"输出前自检:1)覆盖所有块 2)frontmatter完整 3)各小节齐全 4)引用真实 5)篇幅达标 6)语言一致。",
|
|
199
|
+
"",
|
|
200
|
+
"各块精读结果:",
|
|
201
|
+
validReads.map(function (r, i) { return "\\n===== 块 " + (i + 1) + " =====\\n" + r; }).join("\\n")
|
|
202
|
+
]).join("\n");
|
|
203
|
+
|
|
204
|
+
let draft = await agent(draftPrompt, { label: "汇总成稿", phase: "汇总成稿" });
|
|
205
|
+
if (!draft || !draft.trim) return { ok: false, stage: "draft", fatal: false, message: "成稿失败" };
|
|
206
|
+
|
|
207
|
+
// ---------- 5. 质量校验(轻量重试) ----------
|
|
208
|
+
phase("质量校验");
|
|
209
|
+
let verified = draft;
|
|
210
|
+
let passed = false;
|
|
211
|
+
let issues = [];
|
|
212
|
+
for (let round = 0; round <= maxRetries; round++) {
|
|
213
|
+
const checkPrompt = [
|
|
214
|
+
"请对总结笔记做质量校验,逐项检查并输出 JSON:",
|
|
215
|
+
"1.coverage 2.citationsReal 3.terminologyConsistent 4.formatComplete 5.lengthMet(≥" + minWords + "字) 6.languageConsistent",
|
|
216
|
+
"输出:{ \"pass\": true/false, \"issues\": [\"问题\"] },只输出 JSON。",
|
|
217
|
+
"笔记:", verified
|
|
218
|
+
].join("\n");
|
|
219
|
+
const checkSchema = { type: "object", properties: { pass: { type: "boolean" }, issues: { type: "array", items: { type: "string" } } }, required: ["pass","issues"], additionalProperties: false };
|
|
220
|
+
const check = await agent(checkPrompt, { schema: checkSchema, label: "质量校验", phase: "质量校验" });
|
|
221
|
+
passed = !!(check && check.pass);
|
|
222
|
+
issues = (check && check.issues) || [];
|
|
223
|
+
log("校验:" + (passed ? "通过" : "发现问题 " + issues.length + " 条"));
|
|
224
|
+
if (passed) break;
|
|
225
|
+
if (round < maxRetries) {
|
|
226
|
+
const fixPrompt = [
|
|
227
|
+
"修正以下笔记以解决校验问题,输出完整修正版:",
|
|
228
|
+
issues.map(function (s) { return "- " + s; }).join("\n"),
|
|
229
|
+
"当前笔记:", verified
|
|
230
|
+
].join("\n");
|
|
231
|
+
verified = await agent(fixPrompt, { label: "修正笔记", phase: "质量校验" });
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// ---------- 结果 ----------
|
|
236
|
+
return {
|
|
237
|
+
ok: true, kind: kind, title: finalTitle,
|
|
238
|
+
filePath: mdPath, chunksCount: chunkPlan.length, fastMode: fastMode,
|
|
239
|
+
qualityPassed: passed, qualityIssues: issues,
|
|
240
|
+
failedChunks: failedChunks,
|
|
241
|
+
textSource: textSource,
|
|
242
|
+
note: verified
|
|
243
|
+
};
|