deep-read-summarize 0.3.14 → 0.3.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,73 +1,71 @@
1
- {
2
- "name": "deep-read-summarize",
3
- "version": "0.3.14",
4
- "description": "Deep reading & summarization for DSH: books/papers/videos/web → structured Markdown notes (Obsidian / Logseq / static sites). Plugin parsers, MapReduce deep-read, JSON Schema-constrained sub-tasks.",
5
- "type": "commonjs",
6
- "main": "index.js",
7
- "exports": {
8
- ".": "./index.js",
9
- "./workflow": "./workflow.js",
10
- "./parsers": "./parsers/index.js",
11
- "./schemas": "./schemas/index.js",
12
- "./lib/cache": "./lib/cache.js",
13
- "./package.json": "./package.json"
14
- },
15
- "files": [
16
- "index.js",
17
- "workflow.js",
18
- "parsers",
19
- "schemas",
20
- "lib",
21
- "skills",
22
- "scripts",
23
- "tests",
24
- "docs/RELEASE.md",
25
- "docs/RELEASE-v0.1.0.md",
26
- "cordis.patch.yml",
27
- "CHANGELOG.md",
28
- "README.md",
29
- "LICENSE",
30
- "SECURITY.md",
31
- ".env.example"
32
- ],
33
- "scripts": {
34
- "test": "node tests/run-tests.js",
35
- "lint": "node scripts/lint.js",
36
- "validate": "node tests/quick_validate.js && node scripts/security-check.js",
37
- "security": "node scripts/security-check.js",
38
- "test:node": "node --test",
39
- "prepublishOnly": "node tests/run-tests.js && node --test && node scripts/lint.js && node scripts/security-check.js"
40
- },
41
- "dsh": {
42
- "bundle": {
43
- "patch": "./cordis.patch.yml"
44
- },
45
- "lockedRuntime": "workflow tool semantics — see CHANGELOG.md",
46
- "minVersion": "0.1.0",
47
- "breakingPolicy": "re-validate with npm test after each DSH upgrade"
48
- },
49
- "keywords": [
50
- "dsh",
51
- "workflow",
52
- "summarization",
53
- "deep-read",
54
- "obsidian",
55
- "markdown",
56
- "llm",
57
- "subagent",
58
- "mapreduce",
59
- "parsers"
60
- ],
61
- "license": "MIT",
62
- "publishConfig": {
63
- "access": "public"
64
- },
65
- "engines": {
66
- "node": ">=18",
67
- "dsh": ">=0.1.0-rc.6"
68
- },
69
- "repository": {
70
- "type": "git",
71
- "url": "git+https://github.com/PensiveFei/deep-read-summarize.git"
72
- }
73
- }
1
+ {
2
+ "name": "deep-read-summarize",
3
+ "version": "0.3.16",
4
+ "description": "Deep reading & summarization for DSH: books/papers/videos/web → structured Markdown notes (Obsidian / Logseq / static sites). Plugin parsers, MapReduce deep-read, JSON Schema-constrained sub-tasks.",
5
+ "type": "commonjs",
6
+ "main": "index.js",
7
+ "exports": {
8
+ ".": "./index.js",
9
+ "./workflow": "./workflow.js",
10
+ "./parsers": "./parsers/index.js",
11
+ "./schemas": "./schemas/index.js",
12
+ "./lib/cache": "./lib/cache.js",
13
+ "./package.json": "./package.json"
14
+ },
15
+ "files": [
16
+ "index.js",
17
+ "workflow.js",
18
+ "parsers",
19
+ "schemas",
20
+ "lib",
21
+ "skills",
22
+ "scripts",
23
+ "docs/RELEASE.md",
24
+ "cordis.patch.yml",
25
+ "CHANGELOG.md",
26
+ "README.md",
27
+ "LICENSE",
28
+ "SECURITY.md",
29
+ ".env.example"
30
+ ],
31
+ "scripts": {
32
+ "test": "node tests/run-tests.js",
33
+ "lint": "node scripts/lint.js",
34
+ "validate": "node tests/quick_validate.js && node scripts/security-check.js",
35
+ "security": "node scripts/security-check.js",
36
+ "test:node": "node --test",
37
+ "prepublishOnly": "node tests/run-tests.js && node --test && node scripts/lint.js && node scripts/security-check.js"
38
+ },
39
+ "dsh": {
40
+ "bundle": {
41
+ "patch": "./cordis.patch.yml"
42
+ },
43
+ "lockedRuntime": "workflow tool semantics — see CHANGELOG.md",
44
+ "minVersion": "0.1.0",
45
+ "breakingPolicy": "re-validate with npm test after each DSH upgrade"
46
+ },
47
+ "keywords": [
48
+ "dsh",
49
+ "workflow",
50
+ "summarization",
51
+ "deep-read",
52
+ "obsidian",
53
+ "markdown",
54
+ "llm",
55
+ "subagent",
56
+ "mapreduce",
57
+ "parsers"
58
+ ],
59
+ "license": "MIT",
60
+ "publishConfig": {
61
+ "access": "public"
62
+ },
63
+ "engines": {
64
+ "node": ">=18",
65
+ "dsh": ">=0.1.0-rc.6"
66
+ },
67
+ "repository": {
68
+ "type": "git",
69
+ "url": "git+https://github.com/PensiveFei/deep-read-summarize.git"
70
+ }
71
+ }
@@ -1,44 +1,44 @@
1
- // ============================================================================
2
- // parsers/_prompt.js — 共享「获取+分块」提示词模板(内部模块)
3
- // ============================================================================
4
- // 四个内置解析器共用同一提示词骨架,各自只提供差异化配置:
5
- // - label/kindLabel:类型名称(中文 / 中英混合)
6
- // - steps:获取文本的具体步骤(不含分块步骤,模板自动编号)
7
- // - chunkRule:分块规则描述
8
- // - jsonExample:输出 JSON 示例(含 kind 字段)
9
- // 保持各解析器 buildPrompt(input, opts) 插件接口不变;fork 时仍可整体重写。
10
- // ============================================================================
11
-
12
- /**
13
- * 构建「获取+分块」提示词。
14
- * @param {object} cfg
15
- * @param {string} cfg.input 输入(URL 或本地路径)
16
- * @param {object} [cfg.opts] 选项(tempDir/tempFile/maxChunks)
17
- * @param {string} cfg.fetchTarget 获取对象描述(如「书籍全文」「视频字幕/转写」)
18
- * @param {string} cfg.kindLabel 内容类型描述(如「书籍(book)」)
19
- * @param {string[]} cfg.steps 获取步骤(不含分块步骤;模板统一编号并追加分块步骤)
20
- * @param {string} cfg.chunkRule 分块规则(如「章节(或章节内小节)」)
21
- * @param {string} cfg.jsonExample 输出 JSON 示例
22
- * @returns {string} 完整提示词
23
- */
24
- function buildFetchPrompt(cfg) {
25
- const opts = cfg.opts || {};
26
- const tempDir = opts.tempDir || "./.tmp";
27
- const tempFile = opts.tempFile || (tempDir.replace(/[\\/]+$/, "") + "/input.txt");
28
- const maxChunks = opts.maxChunks || 6;
29
- const steps = (cfg.steps || []).map(function (s, i) { return (i + 1) + ". " + s; });
30
- return [
31
- "你的任务:获取" + cfg.fetchTarget + "并保存为纯文本,然后给出分块计划。",
32
- "内容类型:" + cfg.kindLabel,
33
- "输入来源:" + cfg.input,
34
- "目标文件:" + tempFile + "(先创建目录 " + tempDir + ",用 write 工具写入)",
35
- "步骤:"
36
- ].concat(steps, [
37
- (steps.length + 1) + ". 分块:按" + cfg.chunkRule + "切分,每块约 100-200 行或 4000-6000 字,块数不超过 " + maxChunks + " 块;每块给 id/topic/startLine/endLine。",
38
- "输出 JSON:",
39
- cfg.jsonExample,
40
- "若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
41
- ]).join("\n");
42
- }
43
-
44
- module.exports = { buildFetchPrompt };
1
+ // ============================================================================
2
+ // parsers/_prompt.js — 共享「获取+分块」提示词模板(内部模块)
3
+ // ============================================================================
4
+ // 四个内置解析器共用同一提示词骨架,各自只提供差异化配置:
5
+ // - label/kindLabel:类型名称(中文 / 中英混合)
6
+ // - steps:获取文本的具体步骤(不含分块步骤,模板自动编号)
7
+ // - chunkRule:分块规则描述
8
+ // - jsonExample:输出 JSON 示例(含 kind 字段)
9
+ // 保持各解析器 buildPrompt(input, opts) 插件接口不变;fork 时仍可整体重写。
10
+ // ============================================================================
11
+
12
+ /**
13
+ * 构建「获取+分块」提示词。
14
+ * @param {object} cfg
15
+ * @param {string} cfg.input 输入(URL 或本地路径)
16
+ * @param {object} [cfg.opts] 选项(tempDir/tempFile/maxChunks)
17
+ * @param {string} cfg.fetchTarget 获取对象描述(如「书籍全文」「视频字幕/转写」)
18
+ * @param {string} cfg.kindLabel 内容类型描述(如「书籍(book)」)
19
+ * @param {string[]} cfg.steps 获取步骤(不含分块步骤;模板统一编号并追加分块步骤)
20
+ * @param {string} cfg.chunkRule 分块规则(如「章节(或章节内小节)」)
21
+ * @param {string} cfg.jsonExample 输出 JSON 示例
22
+ * @returns {string} 完整提示词
23
+ */
24
+ function buildFetchPrompt(cfg) {
25
+ const opts = cfg.opts || {};
26
+ const tempDir = opts.tempDir || "./.tmp";
27
+ const tempFile = opts.tempFile || (tempDir.replace(/[\\/]+$/, "") + "/input.txt");
28
+ const maxChunks = opts.maxChunks || 6;
29
+ const steps = (cfg.steps || []).map(function (s, i) { return (i + 1) + ". " + s; });
30
+ return [
31
+ "你的任务:获取" + cfg.fetchTarget + "并保存为纯文本,然后给出分块计划。",
32
+ "内容类型:" + cfg.kindLabel,
33
+ "输入来源:" + cfg.input,
34
+ "目标文件:" + tempFile + "(先创建目录 " + tempDir + ",用 write 工具写入)",
35
+ "步骤:"
36
+ ].concat(steps, [
37
+ (steps.length + 1) + ". 分块:按" + cfg.chunkRule + "切分,每块约 100-200 行或 4000-6000 字,块数不超过 " + maxChunks + " 块;每块给 id/topic/startLine/endLine。",
38
+ "输出 JSON:",
39
+ cfg.jsonExample,
40
+ "若无法获取文本输出 { \"saved\": false, \"message\": \"原因\" }。只输出 JSON。"
41
+ ]).join("\n");
42
+ }
43
+
44
+ module.exports = { buildFetchPrompt };
package/scripts/lint.js CHANGED
@@ -1,28 +1,76 @@
1
- // scripts/lint.js — syntax-check all JS files (node --check)
2
- // 跳过 node_modules / 隐藏目录(.git, .tmp* 等)/ 构建与临时目录。
3
- const { execSync } = require('child_process');
4
- const fs = require('fs');
5
- const path = require('path');
1
+ // scripts/lint.js — 语法检查 + 一组零依赖的文本卫生检查
2
+ //
3
+ // 为什么要有后面那一半:本文件此前只跑 `node --check`(纯语法),抓不到
4
+ // 换行不一致、尾随空白、`var` 这些「不影响运行但会制造假 diff / 隐性 bug」的问题。
5
+ // 2026-09-22 与 2026-10-17 两次都被「工作区被写成 CRLF」咬过,而当时的 lint 一声不响。
6
+ //
7
+ // 刻意不做的检查:宽松相等(== / !=)需要真解析器才能在字符串与注释里不误报,
8
+ // 本项目零依赖,强行用正则只会 100% 误报,所以不做。
9
+ const { execSync } = require('node:child_process');
10
+ const fs = require('node:fs');
11
+ const path = require('node:path');
6
12
 
13
+ const ROOT = path.join(__dirname, '..');
7
14
  const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'coverage', '.nyc_output', '_deepread_temp']);
15
+ // 只检查这些文本类型;二进制(png/tgz 等)一律跳过
16
+ const TEXT_EXT = new Set(['.js', '.cjs', '.mjs', '.json', '.yml', '.yaml', '.md', '.txt', '.example', '.gitattributes']);
17
+ const JS_EXT = new Set(['.js', '.cjs', '.mjs']);
8
18
 
9
- function findJs(dir, acc = []) {
19
+ function walk(dir, acc) {
10
20
  for (const f of fs.readdirSync(dir)) {
11
21
  if (f.startsWith('.tmp')) continue; // 临时目录(.tmp* / .tmp-old-*)
12
22
  const p = path.join(dir, f);
13
23
  const st = fs.statSync(p);
14
24
  if (st.isDirectory()) {
15
- if (!SKIP_DIRS.has(f)) findJs(p, acc);
16
- } else if (f.endsWith('.js')) acc.push(p);
25
+ if (!SKIP_DIRS.has(f)) walk(p, acc);
26
+ } else {
27
+ acc.push(p);
28
+ }
17
29
  }
18
30
  return acc;
19
31
  }
20
32
 
21
- const files = findJs(__dirname + '/..');
22
- let failed = 0;
23
- for (const f of files) {
33
+ const all = walk(ROOT, []);
34
+ const textFiles = [];
35
+ const problems = [];
36
+
37
+ for (const f of all) {
38
+ const rel = path.relative(ROOT, f);
39
+ const ext = path.extname(f).toLowerCase();
40
+ const base = path.basename(f).toLowerCase();
41
+ const isText = TEXT_EXT.has(ext) || TEXT_EXT.has(base);
42
+ if (!isText) continue;
43
+ textFiles.push(f);
44
+
45
+ const buf = fs.readFileSync(f);
46
+ if (buf.length >= 3 && buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) {
47
+ problems.push(rel + ': 含 UTF-8 BOM(JSON/YAML 消费方会直接崩)');
48
+ }
49
+ const text = buf.toString('utf8');
50
+ if (text.indexOf('\r') >= 0) {
51
+ problems.push(rel + ': 含 CR(换行必须统一为 LF)');
52
+ }
53
+ const lines = text.split('\n');
54
+ for (let i = 0; i < lines.length; i++) {
55
+ const line = lines[i];
56
+ if (/[ \t]+$/.test(line)) problems.push(rel + ':' + (i + 1) + ': 行尾有空白');
57
+ if (/^\t/.test(line)) problems.push(rel + ':' + (i + 1) + ': 行首用了制表符(用空格)');
58
+ if (JS_EXT.has(ext) && /^\s*var\s+/.test(line)) {
59
+ problems.push(rel + ':' + (i + 1) + ': 用了 var(用 const / let)');
60
+ }
61
+ }
62
+ }
63
+
64
+ // 语法检查(保持原有行为)
65
+ const jsFiles = textFiles.filter(function (f) { return JS_EXT.has(path.extname(f).toLowerCase()); });
66
+ for (const f of jsFiles) {
24
67
  try { execSync('node --check "' + f + '"', { stdio: 'ignore' }); }
25
- catch (e) { console.error('LINT FAIL:', f); failed++; }
68
+ catch (e) { problems.push(path.relative(ROOT, f) + ': node --check 失败'); }
69
+ }
70
+
71
+ if (problems.length) {
72
+ for (const p of problems) console.error('LINT: ' + p);
73
+ console.error('lint failed: ' + problems.length + ' problem(s) in ' + textFiles.length + ' text file(s)');
74
+ process.exit(1);
26
75
  }
27
- if (failed) { console.error(failed + ' file(s) failed'); process.exit(1); }
28
- console.log('Lint OK:', files.length, 'files');
76
+ console.log('Lint OK: ' + textFiles.length + ' text files (' + jsFiles.length + ' JS) — no BOM, LF only, no trailing whitespace, no leading tabs, no var');
@@ -23,7 +23,14 @@ whenToUse: 用户提供一本书、论文(arXiv/PDF)、视频链接(YouTub
23
23
  ## 输入
24
24
 
25
25
  用户提供:内容链接或本地文件路径。
26
- 可选参数:`type`(auto/book/paper/video/web)、`options`(minWords / fastMode / maxChunks / maxRetries / requireCitations / includeTimestamps / transcribe / whisperModel / language / device / outputDir / tempDir)。
26
+ 可选参数:`type`(auto/book/paper/video/web)、`options`(minWords / fastMode / maxChunks / maxRetries / requireCitations / includeTimestamps / transcribe / whisperModel / language / device / outputDir / tempDir / frontmatter)。
27
+
28
+ `options.frontmatter` 决定 frontmatter 用哪套键(默认 `obsidian`,与历史行为一致):
29
+ - `obsidian`:title / aliases / author / year / type / url / tags / created / status —— 配 Dataview 用这套
30
+ - `generic`:把 Hugo 的保留键换掉(`type` → `docType`、`url` → `sourceUrl`、`created` → `date`),可直接放进 Hugo / Jekyll
31
+ - `minimal`:只留 title / date / sourceUrl / tags
32
+
33
+ **为什么要分档**:`type`、`aliases`、`url` 在 Hugo 里是保留键且语义不同(分别决定 layout、URL 重定向、页面路径),把 Obsidian 的写法直接丢给 Hugo 会改变渲染行为,而不是多出几个无害字段。
27
34
 
28
35
  ## 执行步骤
29
36
 
package/workflow.js CHANGED
@@ -35,7 +35,12 @@ const meta = {
35
35
  };
36
36
 
37
37
  // ===== script(复制到 workflow 工具的 script 参数)=====
38
- const RAW_SCRIPT = "// ========== deep-read-summarize v3.0 ==========\n// 3 波次:解析器获取分块 → 并行精读 → 成稿+质量校验\n// 特性:解析器插件化 / 失败纪律(fatal vs 降级) / 成本控制(maxChunks 强制,块数=子代理数)\n\n// YAML 双引号标量:JSON.stringify 的转义规则是 YAML double-quoted 风格的子集,\n// 标题/作者里出现引号时不会再截断 frontmatter(旧版是裸拼 \"...\")。\nfunction yamlStr(v) { return JSON.stringify(v == null ? \"\" : String(v)); }\n\n// ---------- 0. 配置校验(fatal 级错误:必须终止) ----------\nconst input = (args && args.input || \"\").trim();\nconst forcedType = ((args && args.type) || \"auto\").toLowerCase();\nconst options = (args && args.options) || {};\nconst minWords = options.minWords || 2500;\nconst fastMode = !!options.fastMode;\nconst maxChunks = Math.max(1, Math.min(options.maxChunks || 4, 12));\nconst requireCitations = options.requireCitations !== false;\nconst includeTimestamps = !!options.includeTimestamps;\nconst maxRetries = options.maxRetries || 0;\nconst outputDir = options.outputDir || \"./output\";\nconst tempDir = options.tempDir || \"./.tmp\";\nconst tempFile = tempDir.replace(/[\\\\/]+$/, \"\") + \"/input.txt\";\nconst whisperModel = options.whisperModel || \"small\";\nconst transcribeLanguage = options.language || \"zh\";\nconst transcribeDevice = options.device || \"cpu\";\n\n// 幂等不在这里做:脚本沙箱没有文件系统,写不了指纹记录。\n// 需要幂等的调用方请用 lib/cache.js(fingerprint / hasProcessed / markProcessed)。\n\n// ---------- 1. 解析器表:构建时把解析器内联为 __parsers(自包含);\n// 宿主若注入 args._parsers 则优先使用 ----------\nconst parsers = args._parsers || {};\nconst parserNames = Object.keys(parsers);\n\n// 配置校验依赖解析器表,必须放在其后\nif (!input) throw new Error(\"[FATAL] args.input 缺失:请输入内容链接或文件路径\");\nif (parserNames.length === 0) throw new Error(\"[FATAL] 解析器表为空:脚本没有内联任何解析器\");\nif (forcedType !== \"auto\" && parserNames.indexOf(forcedType) === -1)\n throw new Error(\"[FATAL] args.type 非法:\" + forcedType + \"(可用:\" + [\"auto\"].concat(parserNames).join(\"|\") + \")\");\n\nphase(\"解析器选择\");\nconst kindNames = { book: \"书籍\", paper: \"学术论文\", video: \"视频\", web: \"网页/文章\" };\nconst kindFocus = {\n book: \"按章节主题展开核心论点、论证、关键例子、章节间关系,收集关键概念与可引用原文(含页码/章节)\",\n paper: \"四维精读:1)研究问题与方法 2)结论与贡献 3)局限与批判 4)与同类工作比较;收集术语与可引用原文(含章节/段落)\",\n video: \"按主题脉络组织,提取核心论点、证据、例子、可操作建议,收集关键概念与主讲人原话\" + (includeTimestamps ? \"(含时间戳定位)\" : \"(不标时间戳)\"),\n web: \"提取核心论点、结构拆解、关键概念、可引用原文\"\n};\n\n// ---------- 2. 波次1:解析器获取 + 分块 ----------\nphase(\"获取与分块\");\nconst fetchPrompt = (parsers[forcedType !== \"auto\" ? forcedType : \"web\"] || parsers.web || { buildPrompt: function() { return [\"未知解析器\"]; } })\n .buildPrompt(input, { tempDir: tempDir, tempFile: tempFile, maxChunks: maxChunks, transcribe: options.transcribe !== false, whisperModel: whisperModel, language: transcribeLanguage, device: transcribeDevice });\n// fetchSchema 由插件在构建时从 schemas/index.js 注入(单一真相,避免两份 schema 漂移)。\n\nconst fetched = await agent(fetchPrompt, { schema: fetchSchema, label: \"获取+分块\", phase: \"获取与分块\" });\n\n// 失败纪律:内容获取失败 = 可降级错误(返回失败结果,不抛 fatal)\nif (!fetched || !fetched.saved) {\n return { ok: false, stage: \"fetch\", fatal: false, message: (fetched && fetched.message) || \"获取失败(可降级:检查输入或手动提供文本)\" };\n}\n// 分块计划收敛:块数 = 子代理数,计划失控会顶穿 workflow 单次调用时限;\n// 同时挡掉倒置/越界的行号区间(原样透传会让 read 收到 limit<=0)。\nconst rawPlan = (fetched && fetched.chunkPlan) || [];\nif (rawPlan.length === 0) return { ok: false, stage: \"fetch\", fatal: false, message: \"分块计划为空\" };\nconst declaredLines = Math.floor(Number(fetched.totalLines) || 0);\nconst chunkPlan = [];\nfor (let i = 0; i < rawPlan.length && chunkPlan.length < maxChunks; i++) {\n const c = rawPlan[i] || {};\n const s = Math.floor(Number(c.startLine) || 0);\n const e = Math.floor(Number(c.endLine) || 0);\n if (s < 1 || e < s) continue;\n if (declaredLines > 0 && s > declaredLines) continue;\n chunkPlan.push({\n id: chunkPlan.length + 1,\n topic: String(c.topic || (\"块 \" + (chunkPlan.length + 1))),\n startLine: s,\n endLine: declaredLines > 0 ? Math.min(e, declaredLines) : e\n });\n}\nif (chunkPlan.length === 0) return { ok: false, stage: \"fetch\", fatal: false, message: \"分块计划无效:行号区间缺失或非法\" };\nif (chunkPlan.length < rawPlan.length) log(\"分块计划收敛:\" + rawPlan.length + \" -> \" + chunkPlan.length + \" 块(上限 \" + maxChunks + \")\");\n\n// 类型:auto 时只接受解析器表里真实存在的类型,否则退回回退解析器,\n// 避免 \"auto\" 这种非法值流进 frontmatter 的 type / tags。\nlet kind = forcedType;\nif (forcedType === \"auto\") {\n kind = (typeof fetched.kind === \"string\" && parserNames.indexOf(fetched.kind) !== -1)\n ? fetched.kind\n : (parserNames.indexOf(\"web\") !== -1 ? \"web\" : parserNames[0]);\n}\nconst metadata = (fetched && fetched.metadata) || {};\nconst textSource = (metadata && metadata.textSource) || \"unknown\";\nconst srcLabels = { desc: \"视频简介/desc\", subtitle: \"平台字幕\", transcription: \"完整语音转写\", manual: \"用户提供转写文本\" };\nconst srcLabel = srcLabels[textSource] || textSource;\nlog(\"[source] \" + textSource + \" -> \" + srcLabel);\nlog(\"类型:\" + kind + \" 行数:\" + fetched.totalLines + \" 块数:\" + chunkPlan.length);\n\n// ---------- 3. 波次2:并行精读(Map) ----------\nphase(\"并行精读\");\nconst chunkPrompts = chunkPlan.map(function (c) {\n return [\n \"请深度精读以下内容块,输出结构化精读笔记。\",\n \"内容类型:\" + (kindNames[kind] || kind) + \" | 本块主题:\" + c.topic,\n \"精读侧重:\" + (kindFocus[kind] || kindFocus.web),\n \"用 read 工具读取 \" + tempFile + \" 第 \" + c.startLine + \" 到 \" + c.endLine + \" 行(offset=\" + c.startLine + \", limit=\" + (c.endLine - c.startLine + 1) + \"),然后输出:\",\n \"## 本块主题:\" + c.topic,\n \"### 要点(详细展开,保留数据/例子/论证)\",\n \"### 关键概念(术语+解释)\",\n \"### 关键引用(最多2句原文+位置)\" + (requireCitations ? \",必须标注页码/章节/段落位置\" : \"\"),\n \"### 批判观察(如有)\"\n ].join(\"\\n\");\n});\n\nconst reads = [];\nconst readBatch = 3;\nfor (let bi = 0; bi < chunkPrompts.length; bi += readBatch) {\n const batch = chunkPrompts.slice(bi, bi + readBatch);\n const batchReads = await parallel(batch.map(function (p) {\n return function () { return agent(p, { label: \"精读块\", phase: \"并行精读\" }); };\n }));\n for (let k = 0; k < batchReads.length; k++) reads.push(batchReads[k]);\n}\nconst validReads = reads.filter(function (r) { return r && r.trim && r.trim().length > 0; });\n\n// 失败纪律:某块精读失败 = 可降级(跳过该块,标记缺口继续)\nconst failedChunks = chunkPlan.length - validReads.length;\nif (validReads.length === 0) return { ok: false, stage: \"read\", fatal: false, message: \"所有分块精读均失败\" };\nif (failedChunks > 0) log(\"警告:\" + failedChunks + \" 块精读失败,标记缺口继续成稿\");\nlog(\"精读完成:\" + validReads.length + \"/\" + chunkPlan.length);\n\n// ---------- 4. 波次3:成稿 + 内嵌质量自检(Reduce) ----------\nphase(\"汇总成稿\");\nconst todayDate = new Date();\nconst today = todayDate.getFullYear() + \"-\" + (\"0\" + (todayDate.getMonth() + 1)).slice(-2) + \"-\" + (\"0\" + todayDate.getDate()).slice(-2);\n// 标题先折叠换行/制表与连续空白:否则会同时污染 .md 文件名和 frontmatter。\nconst finalTitle = String(metadata.title || \"\").replace(/[\\r\\n\\t]+/g, \" \").replace(/\\s+/g, \" \").trim() || \"未命名内容\";\n// 文件名清洗:去掉 Windows 非法字符;全部非法时兜底,避免写出 ./output/.md 这种隐藏文件。\nconst safeTitle = finalTitle.replace(/[\\\\/:*?\"<>|]/g, \" \").replace(/\\s+/g, \" \").trim() || \"未命名内容\";\nconst mdPath = outputDir.replace(/[\\\\/]+$/, \"\") + \"/\" + safeTitle + \".md\";\n\nconst extraSections = fastMode ? [] : [\n \"\", \"## 5. 批判性分析(论证强处/弱处/可质疑之处)\", \"\",\n \"## 6. 个人思考与应用(与已有知识联系、如何应用、行动要点)\", \"\",\n \"## 7. 延伸阅读(相关主题/作者其他作品/参考文献)\", \"\"\n];\n\nconst draftPrompt = [\n \"将以下多块精读结果整合为完整 Obsidian 精读笔记,只输出 Markdown(含 YAML frontmatter)。\",\n \"模板:\", \"---\",\n 'title: ' + yamlStr(\"《\" + finalTitle + \"》深度精读笔记\"),\n \"aliases: [\" + yamlStr(finalTitle) + \"]\",\n \"author: \" + yamlStr(metadata.author),\n \"year: \" + yamlStr(metadata.year),\n \"type: \" + yamlStr(kind),\n \"url: \" + yamlStr(metadata.url || input),\n \"tags: [精读, \" + (kindNames[kind] || kind) + \"]\",\n \"created: \" + today,\n \"status: 已完成\",\n \"---\",\n \"# 《\" + finalTitle + \"》深度精读笔记\", \"\",\n \"> 一句话核心概括。\", \"\",\n \"## 1. 概述(约500字:主题/背景/核心问题/总体结论)\", \"\",\n \"## 2. 结构拆解(按主题组织,含详细要点;关键结论标注原文出处)\", \"\",\n \"## 3. 关键概念与术语表(表格)\", \"\",\n \"## 4. 关键引用(仅关键结论\" + (requireCitations ? \",必须标注页码/章节/段落\" : \"\") + \")\",\n].concat(extraSections, [\n \"---\",\n \"*笔记整理于 \" + today + \"*\", \"\",\n \"硬性要求:总篇幅≥\" + minWords + \"字;语言与原文一致;术语统一;引用必须真实;视频\" + (includeTimestamps ? \"标注时间戳\" : \"不标注时间戳\") + \"。\",\n \"输出前自检:1)覆盖所有块 2)frontmatter完整 3)各小节齐全 4)引用真实 5)篇幅达标 6)语言一致。\",\n \"\",\n \"各块精读结果:\",\n validReads.map(function (r, i) { return \"\\\\n===== 块 \" + (i + 1) + \" =====\\\\n\" + r; }).join(\"\\\\n\")\n]).join(\"\\n\");\n\nlet draft = await agent(draftPrompt, { label: \"汇总成稿\", phase: \"汇总成稿\" });\nif (!draft || !draft.trim) return { ok: false, stage: \"draft\", fatal: false, message: \"成稿失败\" };\n\n// ---------- 5. 质量校验(轻量重试) ----------\nphase(\"质量校验\");\nlet verified = draft;\nlet passed = false;\nlet issues = [];\nfor (let round = 0; round <= maxRetries; round++) {\n const checkPrompt = [\n \"请对总结笔记做质量校验,逐项检查并输出 JSON:\",\n \"1.coverage 2.citationsReal 3.terminologyConsistent 4.formatComplete 5.lengthMet(≥\" + minWords + \"字) 6.languageConsistent\",\n \"输出:{ \\\"pass\\\": true/false, \\\"issues\\\": [\\\"问题\\\"] },只输出 JSON。\",\n \"笔记:\", verified\n ].join(\"\\n\");\n const checkSchema = { type: \"object\", properties: { pass: { type: \"boolean\" }, issues: { type: \"array\", items: { type: \"string\" } } }, required: [\"pass\",\"issues\"], additionalProperties: false };\n const check = await agent(checkPrompt, { schema: checkSchema, label: \"质量校验\", phase: \"质量校验\" });\n passed = !!(check && check.pass);\n issues = (check && check.issues) || [];\n log(\"校验:\" + (passed ? \"通过\" : \"发现问题 \" + issues.length + \" 条\"));\n if (passed) break;\n if (round < maxRetries) {\n const fixPrompt = [\n \"修正以下笔记以解决校验问题,输出完整修正版:\",\n issues.map(function (s) { return \"- \" + s; }).join(\"\\n\"),\n \"当前笔记:\", verified\n ].join(\"\\n\");\n verified = await agent(fixPrompt, { label: \"修正笔记\", phase: \"质量校验\" });\n }\n}\n\n// ---------- 结果 ----------\nreturn {\n ok: true, kind: kind, title: finalTitle,\n filePath: mdPath, chunksCount: chunkPlan.length, fastMode: fastMode,\n qualityPassed: passed, qualityIssues: issues,\n failedChunks: failedChunks,\n textSource: textSource,\n note: verified\n};";
38
+ // 正文放在 lib/workflow-script.txt,不再做成转义字符串。
39
+ // 这个脚本曾经被 edit / 字符串替换改坏过两次,根因是「转义套转义」(\n 与 \" 各一层)。
40
+ // 现在它是一份普通文本:可以直接读、直接改,不需要任何编码步骤。
41
+ const fs = require("node:fs");
42
+ const path = require("node:path");
43
+ const RAW_SCRIPT = fs.readFileSync(path.join(__dirname, "lib", "workflow-script.txt"), "utf8");
39
44
 
40
45
  // ===== parsers 注册表(运行时注入,实现插件化)=====
41
46
  const parsers = require("./parsers");
@@ -1,61 +0,0 @@
1
- # v0.1.0 — 首个预发布版
2
-
3
- > **锁定的 DeepSeek Harness 版本**:workflow 工具语义(`agent()` / `parallel()` / `phase()` / `log()` / `args`)。
4
- > **兼容性**:0.x 为早期迭代,接口可能变化;升级前重跑 `npm test`。
5
-
6
- ## 发布说明
7
-
8
- deep-read-summarize 是一个给 DSH(DeepSeek Harness)用的精读工作流:输入一本书、一篇论文、一个视频链接或网页,输出一份带 YAML frontmatter 的结构化 Obsidian 笔记。本版本是首个公开预发布版,核心流程已可用,接口细节在 1.0.0 前可能调整。
9
-
10
- ## 新增
11
-
12
- - **四种输入解析器插件**(`parsers/`):书籍(PDF/EPUB/MOBI)、论文(arXiv/PDF/HTML)、视频(yt-dlp 字幕抓取)、网页正文——每种可单独替换,fork 后只改自己的解析器
13
- - **三波次 MapReduce 精读**:解析器获取分块 → 并行子代理精读(Map)→ 合并成稿(Reduce),长内容不溢出上下文
14
- - **JSON Schema 结构化输出**(`schemas/`):各子任务结果受 schema 约束,不合格自动重试
15
- - **幂等缓存**(`lib/cache.js`):对输入指纹去重,已处理过的输入直接跳过,省 token
16
- - **失败分级**:配置错误直接终止(FATAL);内容解析失败降级为缺口标记继续(可降级)
17
- - **质量校验**:关键引用标注出处 + 成稿自检 + 可选重试(`maxRetries`)
18
- - **Obsidian 输出**:YAML frontmatter(type 字段可配合 Dataview 查询),输出目录可配置(`outputDir`)
19
- - **工程配套**:CI(lint + 17 项 fixture 测试 + Gitleaks 密钥扫描)、SECURITY.md、CONTRIBUTING.md、CHANGELOG.md
20
-
21
- ## 修复
22
-
23
- - 长文本获取不再依赖单次上下文(写入临时文件 + 行范围分块)
24
- - 独立质量校验子代理超时问题(合并进成稿波次)
25
- - 硬编码本地路径全部参数化(`outputDir` / `tempDir`),通过 6/6 安全检查
26
-
27
- ## 升级提醒
28
-
29
- - 首个公开版本,无旧版迁移负担
30
- - `options` 参数与 README「用法」示例保持一致即可
31
-
32
- ## 已知问题
33
-
34
- - DSH 为 developer preview,升级后需重跑 `npm test` 验证
35
- - 视频字幕依赖 yt-dlp,未安装时需手动提供转写文本
36
- - 子代理写文件可能受权限限制,workflow 会返回内容由主代理落盘
37
-
38
- ## 验证
39
-
40
- ```bash
41
- npm test # 17 项离线测试(不依赖真实 API)
42
- npm run lint # 13 个 JS 文件语法检查
43
- npm run validate
44
- ```
45
-
46
- **当前功能列表(v0.1.0)**
47
-
48
- | 能力 | 状态 |
49
- |------|------|
50
- | 书籍解析器(PDF/EPUB/MOBI) | ✅ |
51
- | 论文解析器(arXiv/PDF/HTML) | ✅ |
52
- | 视频解析器(yt-dlp 字幕) | ✅ |
53
- | 网页解析器 | ✅ |
54
- | 分块 + 并行精读(MapReduce) | ✅ |
55
- | JSON Schema 输出约束 | ✅ |
56
- | 幂等缓存 | ✅ |
57
- | 失败分级 | ✅ |
58
- | 质量校验循环 | ✅ |
59
- | Obsidian 输出(Dataview 兼容) | ✅ |
60
- | CI + 密钥扫描 | ✅ |
61
- | 自定义解析器插件(custom-parsers/) | ✅ |
@@ -1,117 +0,0 @@
1
- // tests/cache.test.js — idempotency cache tests
2
- const assert = require('assert');
3
- const fs = require('fs');
4
- const os = require('os');
5
- const path = require('path');
6
- const cache = require('../lib/cache');
7
-
8
- let passed = 0, failed = 0;
9
- function test(name, fn) {
10
- try { fn(); passed++; console.log(' ✅', name); }
11
- catch (e) { failed++; console.error(' ❌', name, '—', e.message); }
12
- }
13
-
14
- const tmp = path.join(os.tmpdir(), 'drs-cache-test-' + Date.now());
15
- const cacheFile = path.join(tmp, 'cache.jsonl');
16
-
17
- test('fingerprint is deterministic for same input', () => {
18
- const a = cache.fingerprint('https://arxiv.org/abs/2307.09042');
19
- const b = cache.fingerprint('https://arxiv.org/abs/2307.09042');
20
- assert.strictEqual(a, b);
21
- assert.ok(a && a.length >= 16);
22
- });
23
-
24
- test('fingerprint differs for different inputs', () => {
25
- const a = cache.fingerprint('input-a');
26
- const b = cache.fingerprint('input-b');
27
- assert.notStrictEqual(a, b);
28
- });
29
-
30
- test('fingerprint normalizes URL variants', () => {
31
- const a = cache.fingerprint('https://EXAMPLE.com/doc/');
32
- const b = cache.fingerprint('https://example.com/doc#frag');
33
- assert.strictEqual(a, b, 'URL variants (case/slash/fragment) should share fingerprint');
34
- const c = cache.fingerprint('https://example.com/doc?x=1');
35
- assert.notStrictEqual(a, c, 'different query should produce different fingerprint');
36
- });
37
-
38
- test('fingerprint normalizes trailing empty query (doc vs doc?)', () => {
39
- const a = cache.fingerprint('https://example.com/doc');
40
- const b = cache.fingerprint('https://example.com/doc?');
41
- assert.strictEqual(a, b, 'trailing "?" (empty query) should not create a distinct fingerprint');
42
- assert.strictEqual(cache.normalizeInput('https://example.com/doc?'), 'https://example.com/doc');
43
- });
44
-
45
- test('fingerprint keeps size 0 distinct from absent meta', () => {
46
- // 空文件(size=0)与「无 meta」不应共享指纹;0 值必须参与拼接
47
- const zero = cache.fingerprint('path-to-empty-file', { size: 0, mtimeMs: 123 });
48
- const absent = cache.fingerprint('path-to-empty-file');
49
- assert.notStrictEqual(zero, absent, 'size=0 must not collapse into absent meta');
50
- const five = cache.fingerprint('path-to-empty-file', { size: 5, mtimeMs: 123 });
51
- assert.notStrictEqual(zero, five, 'size 0 vs 5 must differ');
52
- });
53
-
54
- test('normalizeInput keeps non-URL input and root path unchanged', () => {
55
- assert.strictEqual(cache.normalizeInput('C:\\Users\\x\\doc.md'), 'C:\\Users\\x\\doc.md');
56
- assert.strictEqual(cache.normalizeInput('https://example.com/'), 'https://example.com/');
57
- assert.strictEqual(cache.normalizeInput(' '), '');
58
- });
59
-
60
- test('fingerprint includes file meta for local files', () => {
61
- const f = path.join(tmp, 'sample.txt');
62
- fs.mkdirSync(tmp, { recursive: true });
63
- fs.writeFileSync(f, 'hello');
64
- const st = fs.statSync(f);
65
- const a = cache.fingerprint(f, { size: st.size, mtimeMs: st.mtimeMs });
66
- const b = cache.fingerprint(f, { size: st.size + 1, mtimeMs: st.mtimeMs });
67
- assert.notStrictEqual(a, b, 'size change should change fingerprint');
68
- });
69
-
70
- test('hasProcessed returns false for unknown key', () => {
71
- assert.strictEqual(cache.hasProcessed(cacheFile, 'unknown-key'), false);
72
- });
73
-
74
- test('markProcessed then hasProcessed returns true', () => {
75
- const key = cache.fingerprint('https://example.com/doc');
76
- cache.markProcessed(cacheFile, key, '/tmp/out.md');
77
- assert.strictEqual(cache.hasProcessed(cacheFile, key), true);
78
- });
79
-
80
- test('cache file is JSONL (append-only)', () => {
81
- const raw = fs.readFileSync(cacheFile, 'utf8').trim().split('\n');
82
- assert.ok(raw.length >= 1);
83
- const rec = JSON.parse(raw[0]);
84
- assert.ok(rec.key && rec.outputPath && rec.ts);
85
- });
86
-
87
- test('corrupted line is skipped', () => {
88
- fs.appendFileSync(cacheFile, '{broken json}\n', 'utf8');
89
- const map = cache.readCache(cacheFile);
90
- assert.ok(map.size >= 1, 'should still read valid lines');
91
- });
92
-
93
- test('hasProcessed TTL: stale record is treated as unprocessed', () => {
94
- const key = cache.fingerprint('https://example.com/stale');
95
- cache.markProcessed(cacheFile, key, '/tmp/stale.md', { ts: '2020-01-01T00:00:00.000Z' });
96
- assert.strictEqual(cache.hasProcessed(cacheFile, key), true, 'no TTL -> still processed');
97
- assert.strictEqual(cache.hasProcessed(cacheFile, key, { maxAgeMs: 60 * 1000 }), false, 'stale -> unprocessed');
98
- assert.strictEqual(cache.hasProcessed(cacheFile, key, { maxAgeMs: 1e12 }), true, 'recent enough -> processed');
99
- });
100
-
101
- test('markProcessed tolerates unwritable cache location', () => {
102
- const blocker = path.join(tmp, 'blocked-file');
103
- fs.writeFileSync(blocker, 'x');
104
- const badFile = path.join(blocker, 'cache.jsonl');
105
- let threw = false, ret = null;
106
- try { ret = cache.markProcessed(badFile, 'some-key', '/out.md'); } catch (e) { threw = true; }
107
- assert.strictEqual(threw, false, 'should not throw');
108
- assert.strictEqual(ret, false, 'should report failure instead');
109
- });
110
-
111
- // cleanup
112
- fs.rmSync(tmp, { recursive: true, force: true });
113
-
114
- console.log('');
115
- console.log(passed + ' cache tests passed, ' + failed + ' failed');
116
- if (require.main === module) process.exit(failed ? 1 : 0);
117
- module.exports = { passed, failed };
@@ -1,28 +0,0 @@
1
- # The Little Fox and the Stars
2
-
3
- A public-domain style fable by the deep-read-summarize team (CC0 / self-authored for testing).
4
-
5
- ## Chapter 1: The Forest
6
-
7
- Once there was a little fox named Pip who lived in a forest where the stars came down to drink from the river at midnight. Every night, Pip watched them shimmer and dance, and he wished he could join them.
8
-
9
- One evening, a star with a crooked tail stumbled onto the riverbank, too tired to fly back up. "Please," said the star, "help me reach the sky again."
10
-
11
- Pip thought for a moment. The star was heavy and the sky was far. But Pip remembered that the tallest tree in the forest touched the clouds on windy nights.
12
-
13
- ## Chapter 2: The Climb
14
-
15
- Pip carried the star to the Great Oak and began to climb. The branches were slippery with dew, and twice the star nearly fell. "Hold tight," said Pip, "we are almost there."
16
-
17
- At the top, the wind blew strong and the sky opened wide. Pip lifted the star with all his strength, and the star floated up, glowing brighter with each heartbeat.
18
-
19
- ## Chapter 3: The Promise
20
-
21
- "Come back tomorrow night," called the star, "and I will teach you how to shine."
22
-
23
- And Pip did. He learned that even the smallest fox can carry a little light, as long as he climbs one branch at a time.
24
-
25
- THE END
26
-
27
- ---
28
- This fable was written solely for software testing and is released into the public domain (CC0).